1 /*
2 * Copyright (c) 2013, 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
26 #include "gc_implementation/shenandoah/shenandoahAsserts.hpp"
27 #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
29 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc_implementation/shenandoah/shenandoahHeuristics.hpp"
31 #include "runtime/interfaceSupport.hpp"
32
33 class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure {
34 private:
35 ShenandoahHeap* _heap;
36 ShenandoahBarrierSet* _bs;
37
38 template <class T>
39 inline void do_oop_work(T* p) {
40 _heap->maybe_update_with_forwarded(p);
41 }
42 public:
43 ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()), _bs(ShenandoahBarrierSet::barrier_set()) {
44 assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled");
45 }
46 void do_oop(oop* p) { do_oop_work(p); }
47 void do_oop(narrowOop* p) { do_oop_work(p); }
48 };
49
50 ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) :
51 BarrierSet(),
52 _heap(heap)
53 {
54 _kind = BarrierSet::ShenandoahBarrierSet;
55 }
56
57 void ShenandoahBarrierSet::print_on(outputStream* st) const {
58 st->print("ShenandoahBarrierSet");
59 }
60
61 bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) {
62 return bsn == BarrierSet::ShenandoahBarrierSet;
63 }
64
65 bool ShenandoahBarrierSet::has_read_prim_array_opt() {
66 return true;
67 }
68
69 bool ShenandoahBarrierSet::has_read_prim_barrier() {
70 return false;
71 }
72
73 bool ShenandoahBarrierSet::has_read_ref_array_opt() {
74 return true;
75 }
76
77 bool ShenandoahBarrierSet::has_read_ref_barrier() {
78 return false;
79 }
80
81 bool ShenandoahBarrierSet::has_read_region_opt() {
82 return true;
83 }
84
85 bool ShenandoahBarrierSet::has_write_prim_array_opt() {
86 return true;
87 }
88
89 bool ShenandoahBarrierSet::has_write_prim_barrier() {
90 return false;
91 }
92
93 bool ShenandoahBarrierSet::has_write_ref_array_opt() {
94 return true;
95 }
96
97 bool ShenandoahBarrierSet::has_write_ref_barrier() {
98 return true;
99 }
100
101 bool ShenandoahBarrierSet::has_write_ref_pre_barrier() {
102 return true;
103 }
104
105 bool ShenandoahBarrierSet::has_write_region_opt() {
106 return true;
107 }
108
109 bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) {
110 return true;
111 }
112
113 bool ShenandoahBarrierSet::read_prim_needs_barrier(HeapWord* hw, size_t s) {
114 return false;
115 }
116
117 void ShenandoahBarrierSet::read_ref_field(void* v) {
118 // tty->print_cr("read_ref_field: v = "PTR_FORMAT, v);
119 // return *v;
120 }
121
122 template <class T>
123 void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) {
124 assert(UseShenandoahGC && ShenandoahCloneBarrier, "Should be enabled");
125 ShenandoahUpdateRefsForOopClosure cl;
126 T* dst = (T*) start;
127 for (size_t i = 0; i < count; i++) {
128 cl.do_oop(dst++);
129 }
130 }
131
132 void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) {
133 assert(UseShenandoahGC, "should be enabled");
134 if (!ShenandoahCloneBarrier) return;
135 if (!need_update_refs_barrier()) return;
136
137 ShenandoahEvacOOMScope oom_evac_scope;
138 if (UseCompressedOops) {
139 write_ref_array_loop<narrowOop>(start, count);
140 } else {
141 write_ref_array_loop<oop>(start, count);
142 }
143 }
144
145 template <class T>
146 void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, size_t count) {
147 assert (UseShenandoahGC && ShenandoahSATBBarrier, "Should be enabled");
148
149 shenandoah_assert_not_in_cset_loc_except(dst, _heap->cancelled_gc());
150
151 if (! JavaThread::satb_mark_queue_set().is_active()) return;
152 T* elem_ptr = dst;
153 for (size_t i = 0; i < count; i++, elem_ptr++) {
154 T heap_oop = oopDesc::load_heap_oop(elem_ptr);
155 if (!oopDesc::is_null(heap_oop)) {
156 enqueue(oopDesc::decode_heap_oop_not_null(heap_oop));
157 }
158 }
159 }
160
161 void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) {
162 if (! dest_uninitialized && ShenandoahSATBBarrier) {
163 write_ref_array_pre_work(dst, (size_t)count);
164 }
165 }
166
167 void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) {
168 if (! dest_uninitialized && ShenandoahSATBBarrier) {
169 write_ref_array_pre_work(dst, (size_t)count);
170 }
171 }
172
173 template <class T>
174 void ShenandoahBarrierSet::write_ref_field_pre_static(T* field, oop newVal) {
175 T heap_oop = oopDesc::load_heap_oop(field);
176
177 shenandoah_assert_not_in_cset_loc_except(field, ShenandoahHeap::heap()->cancelled_gc());
178
179 if (!oopDesc::is_null(heap_oop)) {
180 ShenandoahBarrierSet::barrier_set()->enqueue(oopDesc::decode_heap_oop(heap_oop));
181 }
182 }
183
184 template <class T>
185 inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) {
186 write_ref_field_pre_static(field, newVal);
187 }
188
189 // These are the more general virtual versions.
190 void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) {
191 write_ref_field_pre_static(field, new_val);
192 }
193
194 void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) {
195 write_ref_field_pre_static(field, new_val);
196 }
197
198 void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) {
199 shenandoah_assert_not_in_cset_loc_except(v, _heap->cancelled_gc());
200 shenandoah_assert_not_forwarded_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress());
201 shenandoah_assert_not_in_cset_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress());
202 }
203
204 void ShenandoahBarrierSet::write_region_work(MemRegion mr) {
205 assert(UseShenandoahGC, "should be enabled");
206 if (!ShenandoahCloneBarrier) return;
207 if (! need_update_refs_barrier()) return;
208
209 // This is called for cloning an object (see jvm.cpp) after the clone
210 // has been made. We are not interested in any 'previous value' because
211 // it would be NULL in any case. But we *are* interested in any oop*
212 // that potentially need to be updated.
213
214 ShenandoahEvacOOMScope oom_evac_scope;
215 oop obj = oop(mr.start());
216 shenandoah_assert_correct(NULL, obj);
217 ShenandoahUpdateRefsForOopClosure cl;
218 obj->oop_iterate(&cl);
219 }
220
221 oop ShenandoahBarrierSet::read_barrier(oop src) {
222 // Check for forwarded objects, because on Full GC path we might deal with
223 // non-trivial fwdptrs that contain Full GC specific metadata. We could check
224 // for is_full_gc_in_progress(), but this also covers the case of stable heap,
225 // which provides a bit of performance improvement.
226 if (ShenandoahReadBarrier && _heap->has_forwarded_objects()) {
227 return ShenandoahBarrierSet::resolve_forwarded(src);
228 } else {
229 return src;
230 }
231 }
232
233 bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) {
234 bool eq = oopDesc::unsafe_equals(obj1, obj2);
235 if (! eq && ShenandoahAcmpBarrier) {
236 OrderAccess::loadload();
237 obj1 = resolve_forwarded(obj1);
238 obj2 = resolve_forwarded(obj2);
239 eq = oopDesc::unsafe_equals(obj1, obj2);
240 }
241 return eq;
242 }
243
244 bool ShenandoahBarrierSet::obj_equals(narrowOop obj1, narrowOop obj2) {
245 return obj_equals(oopDesc::decode_heap_oop(obj1), oopDesc::decode_heap_oop(obj2));
246 }
247
248 JRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_JRT(oopDesc* src))
249 oop result = ShenandoahBarrierSet::barrier_set()->write_barrier_mutator(src);
250 return (oopDesc*) result;
251 JRT_END
252
253 IRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_IRT(oopDesc* src))
254 oop result = ShenandoahBarrierSet::barrier_set()->write_barrier_mutator(src);
255 return (oopDesc*) result;
256 IRT_END
257
258 oop ShenandoahBarrierSet::write_barrier_mutator(oop obj) {
259 assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled");
260 assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION), "evac should be in progress");
261 shenandoah_assert_in_cset(NULL, obj);
262
263 oop fwd = resolve_forwarded_not_null(obj);
264 if (oopDesc::unsafe_equals(obj, fwd)) {
265 ShenandoahEvacOOMScope oom_evac_scope;
266 bool evac;
267
268 Thread* thread = Thread::current();
269 oop res_oop = _heap->evacuate_object(obj, thread, evac);
270
271 // Since we are already here and paid the price of getting through runtime call adapters
272 // and acquiring oom-scope, it makes sense to try and evacuate more adjacent objects,
273 // thus amortizing the overhead. For sparsely live heaps, scan costs easily dominate
274 // total assist costs, and can introduce a lot of evacuation latency. This is why we
275 // only scan for _nearest_ N objects, regardless if they are eligible for evac or not.
276 // The scan itself should also avoid touching the non-marked objects below TAMS, because
277 // their metadata (notably, klasses) may be incorrect already.
278
279 size_t max = ShenandoahEvacAssist;
280 if (max > 0) {
281 ShenandoahMarkingContext* ctx = _heap->complete_marking_context();
282
283 ShenandoahHeapRegion* r = _heap->heap_region_containing(obj);
284 assert(r->is_cset(), "sanity");
285
286 HeapWord* cur = (HeapWord*)obj + obj->size() + ShenandoahBrooksPointer::word_size();
287
288 size_t count = 0;
289 while ((cur < r->top()) && ctx->is_marked(oop(cur)) && (count++ < max)) {
290 oop cur_oop = oop(cur);
291 if (oopDesc::unsafe_equals(cur_oop, resolve_forwarded_not_null(cur_oop))) {
292 _heap->evacuate_object(cur_oop, thread, evac);
293 }
294 cur = cur + cur_oop->size() + ShenandoahBrooksPointer::word_size();
295 }
296 }
297
298 return res_oop;
299 }
300 return fwd;
301 }
302
303 oop ShenandoahBarrierSet::write_barrier(oop obj) {
304 if (ShenandoahWriteBarrier && _heap->has_forwarded_objects()) {
305 if (!oopDesc::is_null(obj)) {
306 bool evac_in_progress = _heap->is_evacuation_in_progress();
307 oop fwd = resolve_forwarded_not_null(obj);
308 if (evac_in_progress &&
309 _heap->in_collection_set(obj) &&
310 oopDesc::unsafe_equals(obj, fwd)) {
311 Thread *t = Thread::current();
312 bool evac;
313 if (t->is_Worker_thread()) {
314 return _heap->evacuate_object(obj, t, evac);
315 } else {
316 ShenandoahEvacOOMScope oom_evac_scope;
317 return _heap->evacuate_object(obj, t, evac);
318 }
319 } else {
320 return fwd;
321 }
322 }
323 }
324 return obj;
325 }
326
327 void ShenandoahBarrierSet::enqueue(oop obj) {
328 // Filter marked objects before hitting the SATB queues. The same predicate would
329 // be used by SATBMQ::filter to eliminate already marked objects downstream, but
330 // filtering here helps to avoid wasteful SATB queueing work to begin with.
331 if (!_heap->requires_marking(obj)) return;
332
333 G1SATBCardTableModRefBS::enqueue(obj);
334 }