35 bool ShenandoahBarrierSet::need_update_refs_barrier() {
36 return _heap->is_update_refs_in_progress() ||
37 _heap->is_concurrent_traversal_in_progress() ||
38 (_heap->is_concurrent_mark_in_progress() && _heap->has_forwarded_objects());
39 }
40
41 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
42 return ShenandoahBrooksPointer::forwardee(p);
43 }
44
45 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
46 if (((HeapWord*) p) != NULL) {
47 return resolve_forwarded_not_null(p);
48 } else {
49 return p;
50 }
51 }
52
53 template <DecoratorSet decorators, typename BarrierSetT>
54 template <typename T>
55 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
56 oop res;
57 oop expected = compare_value;
58 do {
59 compare_value = expected;
60 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
61 expected = res;
62 } while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected)));
63 if (oopDesc::equals_raw(expected, compare_value)) {
64 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
65 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value) &&
66 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
67 ShenandoahBarrierSet::barrier_set()->enqueue(compare_value);
68 }
69 }
70 return res;
71 }
72
73 template <DecoratorSet decorators, typename BarrierSetT>
74 template <typename T>
75 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
76 oop previous = Raw::oop_atomic_xchg(new_value, addr);
77 if (ShenandoahSATBBarrier) {
78 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
79 if (keep_alive && !CompressedOops::is_null(previous) &&
80 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
81 ShenandoahBarrierSet::barrier_set()->enqueue(previous);
82 }
83 }
84 return previous;
85 }
86
87 template <DecoratorSet decorators, typename BarrierSetT>
88 template <typename T>
89 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
90 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
91 size_t length) {
92 if (!CompressedOops::is_null(src_obj)) {
93 src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
94 }
95 if (!CompressedOops::is_null(dst_obj)) {
96 dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
97 }
98 Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
99 }
100
101 template <typename T>
102 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound,
103 bool checkcast, bool satb, bool disjoint,
104 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
105 if (checkcast) {
106 return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, disjoint, storeval_mode);
107 } else {
108 return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, disjoint, storeval_mode);
109 }
110 }
111
112 template <typename T, bool CHECKCAST>
113 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound,
114 bool satb, bool disjoint,
115 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
116 if (satb) {
117 return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, disjoint, storeval_mode);
118 } else {
231 }
232 obj = forw;
233 }
234 enqueue(obj);
235 break;
236 default:
237 ShouldNotReachHere();
238 }
239
240 RawAccess<IS_NOT_NULL>::oop_store(cur_dst, obj);
241 } else {
242 // Store null.
243 RawAccess<>::oop_store(cur_dst, o);
244 }
245 return true;
246 }
247
248 // Clone barrier support
249 template <DecoratorSet decorators, typename BarrierSetT>
250 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
251 src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src));
252 dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst));
253 Raw::clone(src, dst, size);
254 ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));
255 }
256
257 template <DecoratorSet decorators, typename BarrierSetT>
258 template <typename T>
259 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
260 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
261 size_t length) {
262 ShenandoahHeap* heap = ShenandoahHeap::heap();
263 if (!CompressedOops::is_null(src_obj)) {
264 src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
265 }
266 if (!CompressedOops::is_null(dst_obj)) {
267 dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
268 }
269
270 bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();
271 bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;
272 bool disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
273 ArrayCopyStoreValMode storeval_mode;
274 if (heap->has_forwarded_objects()) {
275 if (heap->is_concurrent_traversal_in_progress()) {
276 storeval_mode = WRITE_BARRIER;
277 } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) {
278 storeval_mode = READ_BARRIER;
279 } else {
280 assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress");
281 storeval_mode = NONE; // E.g. during evac or outside cycle
282 }
283 } else {
284 assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress");
285 storeval_mode = NONE;
286 }
287
288 if (!satb && !checkcast && storeval_mode == NONE) {
289 // Short-circuit to bulk copy.
|
35 bool ShenandoahBarrierSet::need_update_refs_barrier() {
36 return _heap->is_update_refs_in_progress() ||
37 _heap->is_concurrent_traversal_in_progress() ||
38 (_heap->is_concurrent_mark_in_progress() && _heap->has_forwarded_objects());
39 }
40
41 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
42 return ShenandoahBrooksPointer::forwardee(p);
43 }
44
45 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
46 if (((HeapWord*) p) != NULL) {
47 return resolve_forwarded_not_null(p);
48 } else {
49 return p;
50 }
51 }
52
53 template <DecoratorSet decorators, typename BarrierSetT>
54 template <typename T>
55 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
56 oop value = Raw::oop_load_in_heap(addr);
57 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
58 keep_alive_if_weak(decorators, value);
59 return value;
60 }
61
62 template <DecoratorSet decorators, typename BarrierSetT>
63 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
64 oop value = Raw::oop_load_in_heap_at(base, offset);
65 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
66 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
67 return value;
68 }
69
70 template <DecoratorSet decorators, typename BarrierSetT>
71 template <typename T>
72 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
73 oop value = Raw::oop_load_not_in_heap(addr);
74 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
75 keep_alive_if_weak(decorators, value);
76 return value;
77 }
78
79 template <DecoratorSet decorators, typename BarrierSetT>
80 template <typename T>
81 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) {
82 ShenandoahBarrierSet::barrier_set()->storeval_barrier(value);
83 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
84 if (keep_alive) {
85 ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value);
86 }
87 Raw::oop_store_in_heap(addr, value);
88 }
89
90 template <DecoratorSet decorators, typename BarrierSetT>
91 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
92 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value);
93 }
94
95 template <DecoratorSet decorators, typename BarrierSetT>
96 template <typename T>
97 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) {
98 oop res;
99 oop expected = compare_value;
100 do {
101 compare_value = expected;
102 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
103 expected = res;
104 } while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected)));
105 if (res != NULL) {
106 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res);
107 } else {
108 return res;
109 }
110 }
111
112 template <DecoratorSet decorators, typename BarrierSetT>
113 template <typename T>
114 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value) {
115 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
116 oop result = oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value);
117 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
118 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) &&
119 oopDesc::equals_raw(result, compare_value) &&
120 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
121 ShenandoahBarrierSet::barrier_set()->enqueue(result);
122 }
123 return result;
124 }
125
126 template <DecoratorSet decorators, typename BarrierSetT>
127 template <typename T>
128 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
129 oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, addr, compare_value);
130 keep_alive_if_weak(decorators, result);
131 return result;
132 }
133
134 template <DecoratorSet decorators, typename BarrierSetT>
135 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
136 oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
137 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result);
138 return result;
139 }
140
141 template <DecoratorSet decorators, typename BarrierSetT>
142 template <typename T>
143 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) {
144 oop previous = Raw::oop_atomic_xchg(new_value, addr);
145 if (previous != NULL) {
146 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous);
147 } else {
148 return previous;
149 }
150 }
151
152 template <DecoratorSet decorators, typename BarrierSetT>
153 template <typename T>
154 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_impl(oop new_value, T* addr) {
155 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
156 oop result = oop_atomic_xchg_not_in_heap(new_value, addr);
157 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
158 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) &&
159 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
160 ShenandoahBarrierSet::barrier_set()->enqueue(result);
161 }
162 return result;
163 }
164
165 template <DecoratorSet decorators, typename BarrierSetT>
166 template <typename T>
167 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
168 oop result = oop_atomic_xchg_in_heap_impl(new_value, addr);
169 keep_alive_if_weak(addr, result);
170 return result;
171 }
172
173 template <DecoratorSet decorators, typename BarrierSetT>
174 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) {
175 oop result = oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset));
176 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result);
177 return result;
178 }
179
180 template <typename T>
181 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound,
182 bool checkcast, bool satb, bool disjoint,
183 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
184 if (checkcast) {
185 return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, disjoint, storeval_mode);
186 } else {
187 return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, disjoint, storeval_mode);
188 }
189 }
190
191 template <typename T, bool CHECKCAST>
192 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound,
193 bool satb, bool disjoint,
194 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
195 if (satb) {
196 return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, disjoint, storeval_mode);
197 } else {
310 }
311 obj = forw;
312 }
313 enqueue(obj);
314 break;
315 default:
316 ShouldNotReachHere();
317 }
318
319 RawAccess<IS_NOT_NULL>::oop_store(cur_dst, obj);
320 } else {
321 // Store null.
322 RawAccess<>::oop_store(cur_dst, o);
323 }
324 return true;
325 }
326
327 // Clone barrier support
328 template <DecoratorSet decorators, typename BarrierSetT>
329 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
330 Raw::clone(src, dst, size);
331 ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));
332 }
333
334 template <DecoratorSet decorators, typename BarrierSetT>
335 template <typename T>
336 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
337 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
338 size_t length) {
339 ShenandoahHeap* heap = ShenandoahHeap::heap();
340 bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();
341 bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;
342 bool disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
343 ArrayCopyStoreValMode storeval_mode;
344 if (heap->has_forwarded_objects()) {
345 if (heap->is_concurrent_traversal_in_progress()) {
346 storeval_mode = WRITE_BARRIER;
347 } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) {
348 storeval_mode = READ_BARRIER;
349 } else {
350 assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress");
351 storeval_mode = NONE; // E.g. during evac or outside cycle
352 }
353 } else {
354 assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress");
355 storeval_mode = NONE;
356 }
357
358 if (!satb && !checkcast && storeval_mode == NONE) {
359 // Short-circuit to bulk copy.
|