1 /*
2 * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIER_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIER_INLINE_HPP
26
27 #include "gc/z/zAddress.inline.hpp"
28 #include "gc/z/zBarrier.hpp"
29 #include "gc/z/zOop.inline.hpp"
30 #include "gc/z/zResurrection.inline.hpp"
31 #include "runtime/atomic.hpp"
32
33 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
34 inline oop ZBarrier::barrier(volatile oop* p, oop o) {
35 uintptr_t addr = ZOop::to_address(o);
36
37 retry:
38 // Fast path
39 if (fast_path(addr)) {
40 return ZOop::to_oop(addr);
41 }
42
43 // Slow path
44 const uintptr_t good_addr = slow_path(addr);
45
46 // Self heal, but only if the address was actually updated by the slow path,
47 // which might not be the case, e.g. when marking through an already good oop.
48 if (p != NULL && good_addr != addr) {
49 const uintptr_t prev_addr = Atomic::cmpxchg(good_addr, (volatile uintptr_t*)p, addr);
50 if (prev_addr != addr) {
51 // Some other thread overwrote the oop. If this oop was updated by a
52 // weak barrier the new oop might not be good, in which case we need
53 // to re-apply this barrier.
54 addr = prev_addr;
55 goto retry;
56 }
57 }
58
59 return ZOop::to_oop(good_addr);
60 }
61
62 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
63 inline oop ZBarrier::weak_barrier(volatile oop* p, oop o) {
64 const uintptr_t addr = ZOop::to_address(o);
65
66 // Fast path
67 if (fast_path(addr)) {
68 // Return the good address instead of the weak good address
69 // to ensure that the currently active heap view is used.
70 return ZOop::to_oop(ZAddress::good_or_null(addr));
71 }
72
73 // Slow path
74 uintptr_t good_addr = slow_path(addr);
75
76 // Self heal unless the address returned from the slow path is null,
77 // in which case resurrection was blocked and we must let the reference
78 // processor clear the oop. Mutators are not allowed to clear oops in
79 // these cases, since that would be similar to calling Reference.clear(),
80 // which would make the reference non-discoverable or silently dropped
81 // by the reference processor.
82 if (p != NULL && good_addr != 0) {
83 // The slow path returns a good/marked address, but we never mark oops
84 // in a weak load barrier so we always self heal with the remapped address.
85 const uintptr_t weak_good_addr = ZAddress::remapped(good_addr);
86 const uintptr_t prev_addr = Atomic::cmpxchg(weak_good_addr, (volatile uintptr_t*)p, addr);
87 if (prev_addr != addr) {
88 // Some other thread overwrote the oop. The new
89 // oop is guaranteed to be weak good or null.
90 assert(ZAddress::is_weak_good_or_null(prev_addr), "Bad weak overwrite");
91
92 // Return the good address instead of the weak good address
93 // to ensure that the currently active heap view is used.
94 good_addr = ZAddress::good_or_null(prev_addr);
95 }
96 }
97
98 return ZOop::to_oop(good_addr);
99 }
100
101 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
102 inline void ZBarrier::root_barrier(oop* p, oop o) {
103 const uintptr_t addr = ZOop::to_address(o);
104
105 // Fast path
106 if (fast_path(addr)) {
107 return;
108 }
109
110 // Slow path
111 const uintptr_t good_addr = slow_path(addr);
112
113 // Non-atomic healing helps speed up root scanning. This is safe to do
114 // since we are always healing roots in a safepoint, which means we are
115 // never racing with mutators modifying roots while we are healing them.
116 // It's also safe in case multiple GC threads try to heal the same root,
117 // since they would always heal the root in the same way and it does not
118 // matter in which order it happens.
119 *p = ZOop::to_oop(good_addr);
120 }
121
122 inline bool ZBarrier::is_null_fast_path(uintptr_t addr) {
123 return ZAddress::is_null(addr);
124 }
125
126 inline bool ZBarrier::is_good_or_null_fast_path(uintptr_t addr) {
127 return ZAddress::is_good_or_null(addr);
128 }
129
130 inline bool ZBarrier::is_weak_good_or_null_fast_path(uintptr_t addr) {
131 return ZAddress::is_weak_good_or_null(addr);
132 }
133
134 inline bool ZBarrier::is_resurrection_blocked(volatile oop* p, oop* o) {
135 const bool is_blocked = ZResurrection::is_blocked();
136
137 // Reload oop after checking the resurrection blocked state. This is
138 // done to prevent a race where we first load an oop, which is logically
139 // null but not yet cleared, then this oop is cleared by the reference
140 // processor and resurrection is unblocked. At this point the mutator
141 // would see the unblocked state and pass this invalid oop through the
142 // normal barrier path, which would incorrectly try to mark this oop.
143 if (p != NULL) {
144 // First assign to reloaded_o to avoid compiler warning about
145 // implicit dereference of volatile oop.
146 const oop reloaded_o = *p;
147 *o = reloaded_o;
148 }
149
150 return is_blocked;
151 }
152
153 //
154 // Load barrier
155 //
156 inline oop ZBarrier::load_barrier_on_oop(oop o) {
157 return load_barrier_on_oop_field_preloaded((oop*)NULL, o);
158 }
159
160 inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) {
161 const oop o = *p;
162 return load_barrier_on_oop_field_preloaded(p, o);
163 }
164
165 inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
166 return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
167 }
168
169 inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) {
170 for (volatile const oop* const end = p + length; p < end; p++) {
171 load_barrier_on_oop_field(p);
172 }
173 }
174
175 inline oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
176 if (is_resurrection_blocked(p, &o)) {
177 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
178 }
179
180 return load_barrier_on_oop_field_preloaded(p, o);
181 }
182
183 inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
184 if (is_resurrection_blocked(p, &o)) {
185 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
186 }
187
188 return load_barrier_on_oop_field_preloaded(p, o);
189 }
190
191 //
192 // Weak load barrier
193 //
194 inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
195 return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o);
196 }
197
198 inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) {
199 return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o);
200 }
201
202 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field(volatile oop* p) {
203 const oop o = *p;
204 return weak_load_barrier_on_weak_oop_field_preloaded(p, o);
205 }
206
207 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
208 if (is_resurrection_blocked(p, &o)) {
209 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
210 }
211
212 return weak_load_barrier_on_oop_field_preloaded(p, o);
213 }
214
215 inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) {
216 return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o);
217 }
218
219 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field(volatile oop* p) {
220 const oop o = *p;
221 return weak_load_barrier_on_phantom_oop_field_preloaded(p, o);
222 }
223
224 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
225 if (is_resurrection_blocked(p, &o)) {
226 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
227 }
228
229 return weak_load_barrier_on_oop_field_preloaded(p, o);
230 }
231
232 //
233 // Is alive barrier
234 //
235 inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) {
236 // Check if oop is logically non-null. This operation
237 // is only valid when resurrection is blocked.
238 assert(ZResurrection::is_blocked(), "Invalid phase");
239 return weak_load_barrier_on_weak_oop(o) != NULL;
240 }
241
242 inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) {
243 // Check if oop is logically non-null. This operation
244 // is only valid when resurrection is blocked.
245 assert(ZResurrection::is_blocked(), "Invalid phase");
246 return weak_load_barrier_on_phantom_oop(o) != NULL;
247 }
248
249 //
250 // Keep alive barrier
251 //
252 inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) {
253 // This operation is only valid when resurrection is blocked.
254 assert(ZResurrection::is_blocked(), "Invalid phase");
255 const oop o = *p;
256 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o);
257 }
258
259 inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) {
260 // This operation is only valid when resurrection is blocked.
261 assert(ZResurrection::is_blocked(), "Invalid phase");
262 const oop o = *p;
263 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
264 }
265
266 //
267 // Mark barrier
268 //
269 inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) {
270 // The fast path only checks for null since the GC worker
271 // threads doing marking wants to mark through good oops.
272 const oop o = *p;
273
274 if (finalizable) {
275 barrier<is_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o);
276 } else {
277 barrier<is_null_fast_path, mark_barrier_on_oop_slow_path>(p, o);
278 }
279 }
280
281 inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) {
282 for (volatile const oop* const end = p + length; p < end; p++) {
283 mark_barrier_on_oop_field(p, finalizable);
284 }
285 }
286
287 inline void ZBarrier::mark_barrier_on_root_oop_field(oop* p) {
288 const oop o = *p;
289 root_barrier<is_good_or_null_fast_path, mark_barrier_on_root_oop_slow_path>(p, o);
290 }
291
292 //
293 // Relocate barrier
294 //
295 inline void ZBarrier::relocate_barrier_on_root_oop_field(oop* p) {
296 const oop o = *p;
297 root_barrier<is_good_or_null_fast_path, relocate_barrier_on_root_oop_slow_path>(p, o);
298 }
299
300 #endif // SHARE_GC_Z_ZBARRIER_INLINE_HPP