1 /*
2 * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
25 #define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
26
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBrooksPointer.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34
35 bool ShenandoahBarrierSet::need_update_refs_barrier() {
36 return _heap->is_update_refs_in_progress() ||
37 _heap->is_concurrent_traversal_in_progress() ||
38 (_heap->is_concurrent_mark_in_progress() && _heap->has_forwarded_objects());
39 }
40
41 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
42 return ShenandoahBrooksPointer::forwardee(p);
43 }
44
45 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
46 if (((HeapWord*) p) != NULL) {
47 return resolve_forwarded_not_null(p);
48 } else {
49 return p;
50 }
51 }
52
53 template <DecoratorSet decorators, typename BarrierSetT>
54 template <typename T>
55 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
56 oop res;
57 oop expected = compare_value;
58 do {
59 compare_value = expected;
60 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
61 expected = res;
62 } while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected)));
63 if (oopDesc::equals_raw(expected, compare_value)) {
64 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
65 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value) &&
66 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
67 ShenandoahBarrierSet::barrier_set()->enqueue(compare_value);
68 }
69 }
70 return res;
71 }
72
73 template <DecoratorSet decorators, typename BarrierSetT>
74 template <typename T>
75 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
76 oop previous = Raw::oop_atomic_xchg(new_value, addr);
77 if (ShenandoahSATBBarrier) {
78 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
79 if (keep_alive && !CompressedOops::is_null(previous) &&
80 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
81 ShenandoahBarrierSet::barrier_set()->enqueue(previous);
82 }
83 }
84 return previous;
85 }
86
87 template <DecoratorSet decorators, typename BarrierSetT>
88 template <typename T>
89 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
90 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
91 size_t length) {
92 if (!CompressedOops::is_null(src_obj)) {
93 src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
94 }
95 if (!CompressedOops::is_null(dst_obj)) {
96 dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
97 }
98 Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
99 }
100
101 template <typename T>
102 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound,
103 bool checkcast, bool satb, bool disjoint,
104 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
105 if (checkcast) {
106 return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, disjoint, storeval_mode);
107 } else {
108 return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, disjoint, storeval_mode);
109 }
110 }
111
112 template <typename T, bool CHECKCAST>
113 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound,
114 bool satb, bool disjoint,
115 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
116 if (satb) {
117 return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, disjoint, storeval_mode);
118 } else {
119 return arraycopy_loop_3<T, CHECKCAST, false>(src, dst, length, bound, disjoint, storeval_mode);
120 }
121 }
122
123 template <typename T, bool CHECKCAST, bool SATB>
124 bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, bool disjoint,
125 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
126 switch (storeval_mode) {
127 case NONE:
128 return arraycopy_loop<T, CHECKCAST, SATB, NONE>(src, dst, length, bound, disjoint);
129 case READ_BARRIER:
130 return arraycopy_loop<T, CHECKCAST, SATB, READ_BARRIER>(src, dst, length, bound, disjoint);
131 case WRITE_BARRIER:
132 return arraycopy_loop<T, CHECKCAST, SATB, WRITE_BARRIER>(src, dst, length, bound, disjoint);
133 default:
134 ShouldNotReachHere();
135 return true; // happy compiler
136 }
137 }
138
139 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE>
140 bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint) {
141 Thread* thread = Thread::current();
142 ShenandoahMarkingContext* ctx = _heap->marking_context();
143 ShenandoahEvacOOMScope oom_evac_scope;
144
145 // We need to handle four cases:
146 //
147 // a) src < dst, conjoint, can only copy backward only
148 // [...src...]
149 // [...dst...]
150 //
151 // b) src < dst, disjoint, can only copy forward, because types may mismatch
152 // [...src...]
153 // [...dst...]
154 //
155 // c) src > dst, conjoint, can copy forward only
156 // [...src...]
157 // [...dst...]
158 //
159 // d) src > dst, disjoint, can only copy forward, because types may mismatch
160 // [...src...]
161 // [...dst...]
162 //
163 if (src > dst || disjoint) {
164 // copy forward:
165 T* cur_src = src;
166 T* cur_dst = dst;
167 T* src_end = src + length;
168 for (; cur_src < src_end; cur_src++, cur_dst++) {
169 if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread, ctx)) {
170 return false;
171 }
172 }
173 } else {
174 // copy backward:
175 T* cur_src = src + length - 1;
176 T* cur_dst = dst + length - 1;
177 for (; cur_src >= src; cur_src--, cur_dst--) {
178 if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread, ctx)) {
179 return false;
180 }
181 }
182 }
183 return true;
184 }
185
186 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE>
187 bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* const thread, ShenandoahMarkingContext* const ctx) {
188 T o = RawAccess<>::oop_load(cur_src);
189
190 if (SATB) {
191 assert(ShenandoahThreadLocalData::satb_mark_queue(thread).is_active(), "Shouldn't be here otherwise");
192 T prev = RawAccess<>::oop_load(cur_dst);
193 if (!CompressedOops::is_null(prev)) {
194 oop prev_obj = CompressedOops::decode_not_null(prev);
195 switch (STOREVAL_MODE) {
196 case NONE:
197 break;
198 case READ_BARRIER:
199 case WRITE_BARRIER:
200 // The write-barrier case cannot really happen. It's traversal-only and traversal
201 // doesn't currently use SATB. And even if it did, it would not be fatal to just do the normal RB here.
202 prev_obj = ShenandoahBarrierSet::resolve_forwarded_not_null(prev_obj);
203 }
204 if (!ctx->is_marked(prev_obj)) {
205 ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(prev_obj);
206 }
207 }
208 }
209
210 if (!CompressedOops::is_null(o)) {
211 oop obj = CompressedOops::decode_not_null(o);
212
213 if (CHECKCAST) {
214 assert(bound != NULL, "need element klass for checkcast");
215 if (!oopDesc::is_instanceof_or_null(obj, bound)) {
216 return false;
217 }
218 }
219
220 switch (STOREVAL_MODE) {
221 case NONE:
222 break;
223 case READ_BARRIER:
224 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
225 break;
226 case WRITE_BARRIER:
227 if (_heap->in_collection_set(obj)) {
228 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
229 if (oopDesc::equals_raw(forw, obj)) {
230 forw = _heap->evacuate_object(forw, thread);
231 }
232 obj = forw;
233 }
234 enqueue(obj);
235 break;
236 default:
237 ShouldNotReachHere();
238 }
239
240 RawAccess<IS_NOT_NULL>::oop_store(cur_dst, obj);
241 } else {
242 // Store null.
243 RawAccess<>::oop_store(cur_dst, o);
244 }
245 return true;
246 }
247
248 // Clone barrier support
249 template <DecoratorSet decorators, typename BarrierSetT>
250 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
251 src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src));
252 dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst));
253 Raw::clone(src, dst, size);
254 ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));
255 }
256
257 template <DecoratorSet decorators, typename BarrierSetT>
258 template <typename T>
259 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
260 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
261 size_t length) {
262 ShenandoahHeap* heap = ShenandoahHeap::heap();
263 if (!CompressedOops::is_null(src_obj)) {
264 src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
265 }
266 if (!CompressedOops::is_null(dst_obj)) {
267 dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
268 }
269
270 bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();
271 bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;
272 bool disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
273 ArrayCopyStoreValMode storeval_mode;
274 if (heap->has_forwarded_objects()) {
275 if (heap->is_concurrent_traversal_in_progress()) {
276 storeval_mode = WRITE_BARRIER;
277 } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) {
278 storeval_mode = READ_BARRIER;
279 } else {
280 assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress");
281 storeval_mode = NONE; // E.g. during evac or outside cycle
282 }
283 } else {
284 assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress");
285 storeval_mode = NONE;
286 }
287
288 if (!satb && !checkcast && storeval_mode == NONE) {
289 // Short-circuit to bulk copy.
290 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
291 }
292
293 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
294 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
295
296 Klass* bound = objArrayOop(dst_obj)->element_klass();
297 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
298 return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, disjoint, storeval_mode);
299 }
300
301 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP