1 /*
2 * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
26
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBrooksPointer.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31
32 bool ShenandoahBarrierSet::need_update_refs_barrier() {
33 return _heap->is_update_refs_in_progress() ||
34 _heap->is_concurrent_traversal_in_progress() ||
35 (_heap->is_concurrent_mark_in_progress() && _heap->has_forwarded_objects());
36 }
37
38 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
39 return ShenandoahBrooksPointer::forwardee(p);
40 }
41
42 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
43 if (((HeapWord*) p) != NULL) {
44 return resolve_forwarded_not_null(p);
45 } else {
46 return p;
47 }
48 }
49
50 template <DecoratorSet decorators, typename BarrierSetT>
51 template <typename T>
52 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
53 oop res;
54 oop expected = compare_value;
55 do {
56 compare_value = expected;
57 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
58 expected = res;
59 } while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected)));
60 if (oopDesc::equals_raw(expected, compare_value)) {
61 if (ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) {
62 ShenandoahBarrierSet::barrier_set()->enqueue(compare_value);
63 }
64 }
65 return res;
66 }
67
68 template <DecoratorSet decorators, typename BarrierSetT>
69 template <typename T>
70 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
71 oop previous = Raw::oop_atomic_xchg(new_value, addr);
72 if (ShenandoahSATBBarrier) {
73 if (!CompressedOops::is_null(previous)) {
74 ShenandoahBarrierSet::barrier_set()->enqueue(previous);
75 }
76 }
77 return previous;
78 }
79
80 template <DecoratorSet decorators, typename BarrierSetT>
81 template <typename T>
82 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
83 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
84 size_t length) {
85 if (!CompressedOops::is_null(src_obj)) {
86 src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
87 }
88 if (!CompressedOops::is_null(dst_obj)) {
89 dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
90 }
91 Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
92 }
93
94 template <typename T>
95 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound,
96 bool checkcast, bool satb, bool disjoint,
97 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
98 if (checkcast) {
99 return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, disjoint, storeval_mode);
100 } else {
101 return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, disjoint, storeval_mode);
102 }
103 }
104
105 template <typename T, bool CHECKCAST>
106 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound,
107 bool satb, bool disjoint,
108 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
109 if (satb) {
110 return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, disjoint, storeval_mode);
111 } else {
112 return arraycopy_loop_3<T, CHECKCAST, false>(src, dst, length, bound, disjoint, storeval_mode);
113 }
114 }
115
116 template <typename T, bool CHECKCAST, bool SATB>
117 bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, bool disjoint,
118 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
119 switch (storeval_mode) {
120 case NONE:
121 return arraycopy_loop<T, CHECKCAST, SATB, NONE>(src, dst, length, bound, disjoint);
122 case READ_BARRIER:
123 return arraycopy_loop<T, CHECKCAST, SATB, READ_BARRIER>(src, dst, length, bound, disjoint);
124 case WRITE_BARRIER:
125 return arraycopy_loop<T, CHECKCAST, SATB, WRITE_BARRIER>(src, dst, length, bound, disjoint);
126 default:
127 ShouldNotReachHere();
128 return true; // happy compiler
129 }
130 }
131
132 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE>
133 bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint) {
134 Thread* thread = Thread::current();
135
136 ShenandoahEvacOOMScope oom_evac_scope;
137
138 // We need to handle four cases:
139 //
140 // a) src < dst, conjoint, can only copy backward only
141 // [...src...]
142 // [...dst...]
143 //
144 // b) src < dst, disjoint, can only copy forward, because types may mismatch
145 // [...src...]
146 // [...dst...]
147 //
148 // c) src > dst, conjoint, can copy forward only
149 // [...src...]
150 // [...dst...]
151 //
152 // d) src > dst, disjoint, can only copy forward, because types may mismatch
153 // [...src...]
154 // [...dst...]
155 //
156 if (src > dst || disjoint) {
157 // copy forward:
158 T* cur_src = src;
159 T* cur_dst = dst;
160 T* src_end = src + length;
161 for (; cur_src < src_end; cur_src++, cur_dst++) {
162 if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread)) {
163 return false;
164 }
165 }
166 } else {
167 // copy backward:
168 T* cur_src = src + length - 1;
169 T* cur_dst = dst + length - 1;
170 for (; cur_src >= src; cur_src--, cur_dst--) {
171 if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread)) {
172 return false;
173 }
174 }
175 }
176 return true;
177 }
178
179 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE>
180 bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread) {
181 T o = RawAccess<>::oop_load(cur_src);
182
183 if (SATB) {
184 T prev = RawAccess<>::oop_load(cur_dst);
185 if (!CompressedOops::is_null(prev)) {
186 oop prev_obj = CompressedOops::decode_not_null(prev);
187 enqueue(prev_obj);
188 }
189 }
190
191 if (!CompressedOops::is_null(o)) {
192 oop obj = CompressedOops::decode_not_null(o);
193
194 if (CHECKCAST) {
195 assert(bound != NULL, "need element klass for checkcast");
196 if (!oopDesc::is_instanceof_or_null(obj, bound)) {
197 return false;
198 }
199 }
200
201 switch (STOREVAL_MODE) {
202 case NONE:
203 break;
204 case READ_BARRIER:
205 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
206 break;
207 case WRITE_BARRIER:
208 if (_heap->in_collection_set(obj)) {
209 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
210 if (oopDesc::equals_raw(forw, obj)) {
211 forw = _heap->evacuate_object(forw, thread);
212 }
213 obj = forw;
214 }
215 enqueue(obj);
216 break;
217 default:
218 ShouldNotReachHere();
219 }
220
221 RawAccess<IS_NOT_NULL>::oop_store(cur_dst, obj);
222 } else {
223 // Store null.
224 RawAccess<>::oop_store(cur_dst, o);
225 }
226 return true;
227 }
228
229 // Clone barrier support
230 template <DecoratorSet decorators, typename BarrierSetT>
231 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
232 src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src));
233 dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst));
234 Raw::clone(src, dst, size);
235 ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));
236 }
237
238 template <DecoratorSet decorators, typename BarrierSetT>
239 template <typename T>
240 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
241 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
242 size_t length) {
243 ShenandoahHeap* heap = ShenandoahHeap::heap();
244 if (!CompressedOops::is_null(src_obj)) {
245 src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
246 }
247 if (!CompressedOops::is_null(dst_obj)) {
248 dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
249 }
250
251 bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();
252 bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;
253 bool disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
254 ArrayCopyStoreValMode storeval_mode;
255 if (heap->has_forwarded_objects()) {
256 if (heap->is_concurrent_traversal_in_progress()) {
257 storeval_mode = WRITE_BARRIER;
258 } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) {
259 storeval_mode = READ_BARRIER;
260 } else {
261 assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress");
262 storeval_mode = NONE; // E.g. during evac or outside cycle
263 }
264 } else {
265 assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress");
266 storeval_mode = NONE;
267 }
268
269 if (!satb && !checkcast && storeval_mode == NONE) {
270 // Short-circuit to bulk copy.
271 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
272 }
273
274 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
275 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
276
277 Klass* bound = objArrayOop(dst_obj)->element_klass();
278 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
279 return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, disjoint, storeval_mode);
280 }
281
282 #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP