1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_INLINE_HPP
25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_INLINE_HPP
26
27 #include "gc/shared/accessBarrierSupport.hpp"
28 #include "gc/shenandoah/shenandoahLRBBarrierSet.hpp"
29
30 template <DecoratorSet decorators, typename BarrierSetT>
31 template <typename T>
32 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
33 oop obj = Raw::oop_load_in_heap(addr);
34 obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj);
35 ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj);
36 return obj;
37 }
38
39 template <DecoratorSet decorators, typename BarrierSetT>
40 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
41 oop obj = Raw::oop_load_in_heap_at(base, offset);
42 obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj);
43 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), obj);
44 return obj;
45 }
46
47 template <DecoratorSet decorators, typename BarrierSetT>
48 template <typename T>
49 inline void ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) {
50 ShenandoahBaseBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value);
51 ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(value);
52 Raw::oop_store_in_heap(addr, value);
53 }
54
55 template <DecoratorSet decorators, typename BarrierSetT>
56 template <typename T>
57 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
58 ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(new_value);
59
60 oop res;
61 oop expected = compare_value;
62 do {
63 compare_value = expected;
64 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
65 expected = res;
66 } while ((! oopDesc::unsafe_equals(compare_value, expected)) && oopDesc::unsafe_equals(resolve_forwarded(compare_value), resolve_forwarded(expected)));
67 if (oopDesc::unsafe_equals(expected, compare_value)) {
68 if (ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) {
69 ShenandoahBarrierSet::barrier_set()->enqueue(compare_value);
70 }
71 }
72
73 res = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(res);
74 ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, res);
75 return res;
76 }
77
78 template <DecoratorSet decorators, typename BarrierSetT>
79 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
80 ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(new_value);
81
82 oop res;
83 oop expected = compare_value;
84 do {
85 compare_value = expected;
86 res = Raw::oop_atomic_cmpxchg_at(new_value, base, offset, compare_value);
87 expected = res;
88 } while ((! oopDesc::unsafe_equals(compare_value, expected)) && oopDesc::unsafe_equals(resolve_forwarded(compare_value), resolve_forwarded(expected)));
89 if (oopDesc::unsafe_equals(expected, compare_value)) {
90 if (ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) {
91 ShenandoahBarrierSet::barrier_set()->enqueue(compare_value);
92 }
93 }
94
95 res = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(res);
96 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), res);
97 return res;
98 }
99
100 template <DecoratorSet decorators, typename BarrierSetT>
101 template <typename T>
102 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
103 ShenandoahBaseBarrierSet::barrier_set()->write_ref_field_pre_work(addr, new_value);
104 ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(new_value);
105 oop obj = Raw::oop_atomic_xchg_in_heap(new_value, addr);
106 obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj);
107 ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj);
108 return obj;
109 }
110
111 template <DecoratorSet decorators, typename BarrierSetT>
112 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) {
113 ShenandoahBaseBarrierSet::barrier_set()->write_ref_field_pre_work(AccessInternal::oop_field_addr<decorators>(base, offset), new_value);
114 ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(new_value);
115 oop obj = Raw::oop_atomic_xchg_in_heap_at(new_value, base, offset);
116 obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj);
117 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), obj);
118 return obj;
119 }
120
121 template <DecoratorSet decorators, typename BarrierSetT>
122 template <typename T>
123 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
124 oop obj = Raw::oop_load_not_in_heap(addr);
125 obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj);
126 ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj);
127 return obj;
128 }
129
130 template <DecoratorSet decorators, typename BarrierSetT>
131 template <typename T>
132 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) {
133 oop obj = Raw::oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value);
134 obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj);
135 ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj);
136 return obj;
137 }
138
139 template <DecoratorSet decorators, typename BarrierSetT>
140 template <typename T>
141 inline oop ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) {
142 oop obj = Raw::oop_atomic_xchg_not_in_heap(new_value, addr);
143 obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj);
144 ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj);
145 return obj;
146 }
147
148 template <DecoratorSet decorators, typename BarrierSetT>
149 template <typename T>
150 bool ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
151 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
152 size_t length) {
153 ShenandoahHeap* heap = ShenandoahHeap::heap();
154
155 bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();
156 bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;
157 ArrayCopyStoreValMode storeval_mode;
158 if (heap->has_forwarded_objects()) {
159 if (heap->is_concurrent_traversal_in_progress()) {
160 storeval_mode = WRITE_BARRIER;
161 } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) {
162 storeval_mode = READ_BARRIER;
163 } else {
164 assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress");
165 storeval_mode = NONE; // E.g. during evac or outside cycle
166 }
167 } else {
168 assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress");
169 storeval_mode = NONE;
170 }
171
172 if (!satb && !checkcast && storeval_mode == NONE) {
173 // Short-circuit to bulk copy.
174 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
175 }
176
177 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
178 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
179
180 Klass* bound = objArrayOop(dst_obj)->element_klass();
181 ShenandoahBaseBarrierSet* bs = ShenandoahBaseBarrierSet::barrier_set();
182 return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, storeval_mode);
183 }
184
185 template <DecoratorSet decorators, typename BarrierSetT>
186 void ShenandoahLRBBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
187 Raw::clone(src, dst, size);
188 ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));
189 }
190
191 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_INLINE_HPP