1 /*
2 * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIER_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIER_INLINE_HPP
26
27 #include "gc/z/zAddress.inline.hpp"
28 #include "gc/z/zBarrier.hpp"
29 #include "gc/z/zOop.inline.hpp"
30 #include "gc/z/zResurrection.inline.hpp"
31 #include "runtime/atomic.hpp"
32
33 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
34 inline oop ZBarrier::barrier(volatile oop* p, oop o) {
35 uintptr_t addr = ZOop::to_address(o);
36
37 retry:
38 // Fast path
39 if (fast_path(addr)) {
40 return ZOop::to_oop(addr);
41 }
42
43 // Slow path
44 const uintptr_t good_addr = slow_path(addr);
45
46 // Self heal, but only if the address was actually updated by the slow path,
47 // which might not be the case, e.g. when marking through an already good oop.
48 if (p != NULL && good_addr != addr) {
49 const uintptr_t prev_addr = Atomic::cmpxchg(good_addr, (volatile uintptr_t*)p, addr);
50 if (prev_addr != addr) {
51 // Some other thread overwrote the oop. If this oop was updated by a
52 // weak barrier the new oop might not be good, in which case we need
53 // to re-apply this barrier.
54 addr = prev_addr;
55 goto retry;
56 }
57 }
58
59 return ZOop::to_oop(good_addr);
60 }
61
62 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
63 inline oop ZBarrier::weak_barrier(volatile oop* p, oop o) {
64 const uintptr_t addr = ZOop::to_address(o);
65
66 // Fast path
67 if (fast_path(addr)) {
68 // Return the good address instead of the weak good address
69 // to ensure that the currently active heap view is used.
70 return ZOop::to_oop(ZAddress::good_or_null(addr));
71 }
72
73 // Slow path
74 uintptr_t good_addr = slow_path(addr);
75
76 // Self heal unless the address returned from the slow path is null,
77 // in which case resurrection was blocked and we must let the reference
78 // processor clear the oop. Mutators are not allowed to clear oops in
79 // these cases, since that would be similar to calling Reference.clear(),
80 // which would make the reference non-discoverable or silently dropped
81 // by the reference processor.
82 if (p != NULL && good_addr != 0) {
83 // The slow path returns a good/marked address, but we never mark oops
84 // in a weak load barrier so we always self heal with the remapped address.
85 const uintptr_t weak_good_addr = ZAddress::remapped(good_addr);
86 const uintptr_t prev_addr = Atomic::cmpxchg(weak_good_addr, (volatile uintptr_t*)p, addr);
87 if (prev_addr != addr) {
88 // Some other thread overwrote the oop. The new
89 // oop is guaranteed to be weak good or null.
90 assert(ZAddress::is_weak_good_or_null(prev_addr), "Bad weak overwrite");
91
92 // Return the good address instead of the weak good address
93 // to ensure that the currently active heap view is used.
94 good_addr = ZAddress::good_or_null(prev_addr);
95 }
96 }
97
98 return ZOop::to_oop(good_addr);
99 }
100
101 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
102 inline void ZBarrier::root_barrier(volatile oop* p, oop o) {
103 const uintptr_t addr = ZOop::to_address(o);
104
105 // Fast path
106 if (fast_path(addr)) {
107 return;
108 }
109
110 // Slow path
111 const uintptr_t good_addr = slow_path(addr);
112
113 // Non-atomic healing helps speed up root scanning. This is safe to do
114 // since we are always healing roots in a safepoint, which means we are
115 // never racing with mutators modifying roots while we are healing them.
116 // It's also safe in case multiple GC threads try to heal the same root,
117 // since they would always heal the root in the same way and it does not
118 // matter in which order it happens.
119 *p = ZOop::to_oop(good_addr);
120 }
121
122 inline bool ZBarrier::is_null_fast_path(uintptr_t addr) {
123 return ZAddress::is_null(addr);
124 }
125
126 inline bool ZBarrier::is_good_or_null_fast_path(uintptr_t addr) {
127 return ZAddress::is_good_or_null(addr);
128 }
129
130 inline bool ZBarrier::is_weak_good_or_null_fast_path(uintptr_t addr) {
131 return ZAddress::is_weak_good_or_null(addr);
132 }
133
134 inline bool ZBarrier::is_resurrection_blocked(volatile oop* p, oop* o) {
135 const bool is_blocked = ZResurrection::is_blocked();
136
137 // Reload oop after checking the resurrection blocked state. This is
138 // done to prevent a race where we first load an oop, which is logically
139 // null but not yet cleared, then this oop is cleared by the reference
140 // processor and resurrection is unblocked. At this point the mutator
141 // would see the unblocked state and pass this invalid oop through the
142 // normal barrier path, which would incorrectly try to mark this oop.
143 if (p != NULL) {
144 *o = *p;
145 }
146
147 return is_blocked;
148 }
149
150 //
151 // Load barrier
152 //
153 inline oop ZBarrier::load_barrier_on_oop(oop o) {
154 return load_barrier_on_oop_field_preloaded((oop*)NULL, o);
155 }
156
157 inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) {
158 const oop o = *p;
159 return load_barrier_on_oop_field_preloaded(p, o);
160 }
161
162 inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
163 return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
164 }
165
166 inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) {
167 for (volatile const oop* const end = p + length; p < end; p++) {
168 load_barrier_on_oop_field(p);
169 }
170 }
171
172 inline oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
173 if (is_resurrection_blocked(p, &o)) {
174 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
175 }
176
177 return load_barrier_on_oop_field_preloaded(p, o);
178 }
179
180 inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
181 if (is_resurrection_blocked(p, &o)) {
182 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
183 }
184
185 return load_barrier_on_oop_field_preloaded(p, o);
186 }
187
188 //
189 // Weak load barrier
190 //
191 inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
192 return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o);
193 }
194
195 inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) {
196 return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o);
197 }
198
199 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field(volatile oop* p) {
200 const oop o = *p;
201 return weak_load_barrier_on_weak_oop_field_preloaded(p, o);
202 }
203
204 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
205 if (is_resurrection_blocked(p, &o)) {
206 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
207 }
208
209 return weak_load_barrier_on_oop_field_preloaded(p, o);
210 }
211
212 inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) {
213 return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o);
214 }
215
216 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field(volatile oop* p) {
217 const oop o = *p;
218 return weak_load_barrier_on_phantom_oop_field_preloaded(p, o);
219 }
220
221 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
222 if (is_resurrection_blocked(p, &o)) {
223 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
224 }
225
226 return weak_load_barrier_on_oop_field_preloaded(p, o);
227 }
228
229 //
230 // Is alive barrier
231 //
232 inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) {
233 // Check if oop is logically non-null. This operation
234 // is only valid when resurrection is blocked.
235 assert(ZResurrection::is_blocked(), "Invalid phase");
236 return weak_load_barrier_on_weak_oop(o) != NULL;
237 }
238
239 inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) {
240 // Check if oop is logically non-null. This operation
241 // is only valid when resurrection is blocked.
242 assert(ZResurrection::is_blocked(), "Invalid phase");
243 return weak_load_barrier_on_phantom_oop(o) != NULL;
244 }
245
246 //
247 // Keep alive barrier
248 //
249 inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) {
250 // This operation is only valid when resurrection is blocked.
251 assert(ZResurrection::is_blocked(), "Invalid phase");
252 const oop o = *p;
253 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o);
254 }
255
256 inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) {
257 // This operation is only valid when resurrection is blocked.
258 assert(ZResurrection::is_blocked(), "Invalid phase");
259 const oop o = *p;
260 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
261 }
262
263 //
264 // Mark barrier
265 //
266 inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) {
267 // The fast path only checks for null since the GC worker
268 // threads doing marking wants to mark through good oops.
269 const oop o = *p;
270
271 if (finalizable) {
272 barrier<is_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o);
273 } else {
274 barrier<is_null_fast_path, mark_barrier_on_oop_slow_path>(p, o);
275 }
276 }
277
278 inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) {
279 for (volatile const oop* const end = p + length; p < end; p++) {
280 mark_barrier_on_oop_field(p, finalizable);
281 }
282 }
283
284 inline void ZBarrier::mark_barrier_on_root_oop_field(volatile oop* p) {
285 const oop o = *p;
286 root_barrier<is_good_or_null_fast_path, mark_barrier_on_root_oop_slow_path>(p, o);
287 }
288
289 //
290 // Relocate barrier
291 //
292 inline void ZBarrier::relocate_barrier_on_root_oop_field(volatile oop* p) {
293 const oop o = *p;
294 root_barrier<is_good_or_null_fast_path, relocate_barrier_on_root_oop_slow_path>(p, o);
295 }
296
297 #endif // SHARE_GC_Z_ZBARRIER_INLINE_HPP