1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
28 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "runtime/interfaceSupport.hpp"
31
32 template <bool UPDATE_MATRIX, bool STOREVAL_WRITE_BARRIER>
33 class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure {
34 private:
35 ShenandoahHeap* _heap;
36 template <class T>
37 inline void do_oop_work(T* p) {
38 oop o;
39 if (STOREVAL_WRITE_BARRIER) {
40 bool evac;
41 o = _heap->evac_update_oop_ref(p, evac);
42 if (evac) {
43 G1SATBCardTableModRefBS::enqueue(o);
44 }
45 } else {
46 o = _heap->maybe_update_oop_ref(p);
47 }
48 if (UPDATE_MATRIX && !oopDesc::is_null(o)) {
49 _heap->connection_matrix()->set_connected(p, o);
50 }
51 }
52 public:
53 ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()) {}
54 void do_oop(oop* p) { do_oop_work(p); }
55 void do_oop(narrowOop* p) { do_oop_work(p); }
56 };
57
58 ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) :
59 BarrierSet(BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)),
60 _heap(heap)
61 {
62 }
63
64 void ShenandoahBarrierSet::print_on(outputStream* st) const {
65 st->print("ShenandoahBarrierSet");
66 }
67
68 bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) {
69 return bsn == BarrierSet::ShenandoahBarrierSet;
70 }
71
72 bool ShenandoahBarrierSet::has_read_prim_array_opt() {
73 return true;
74 }
75
76 bool ShenandoahBarrierSet::has_read_prim_barrier() {
77 return false;
78 }
79
80 bool ShenandoahBarrierSet::has_read_ref_array_opt() {
81 return true;
82 }
83
84 bool ShenandoahBarrierSet::has_read_ref_barrier() {
85 return false;
86 }
87
88 bool ShenandoahBarrierSet::has_read_region_opt() {
89 return true;
90 }
91
92 bool ShenandoahBarrierSet::has_write_prim_array_opt() {
93 return true;
94 }
95
96 bool ShenandoahBarrierSet::has_write_prim_barrier() {
97 return false;
98 }
99
100 bool ShenandoahBarrierSet::has_write_ref_array_opt() {
101 return true;
102 }
103
104 bool ShenandoahBarrierSet::has_write_ref_barrier() {
105 return true;
106 }
107
108 bool ShenandoahBarrierSet::has_write_ref_pre_barrier() {
109 return true;
110 }
111
112 bool ShenandoahBarrierSet::has_write_region_opt() {
113 return true;
114 }
115
116 bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) {
117 return true;
118 }
119
120 void ShenandoahBarrierSet::read_prim_array(MemRegion mr) {
121 Unimplemented();
122 }
123
124 void ShenandoahBarrierSet::read_prim_field(HeapWord* hw, size_t s){
125 Unimplemented();
126 }
127
128 bool ShenandoahBarrierSet::read_prim_needs_barrier(HeapWord* hw, size_t s) {
129 return false;
130 }
131
132 void ShenandoahBarrierSet::read_ref_array(MemRegion mr) {
133 Unimplemented();
134 }
135
136 void ShenandoahBarrierSet::read_ref_field(void* v) {
137 // tty->print_cr("read_ref_field: v = "PTR_FORMAT, v);
138 // return *v;
139 }
140
141 bool ShenandoahBarrierSet::read_ref_needs_barrier(void* v) {
142 Unimplemented();
143 return false;
144 }
145
146 void ShenandoahBarrierSet::read_region(MemRegion mr) {
147 Unimplemented();
148 }
149
150 void ShenandoahBarrierSet::resize_covered_region(MemRegion mr) {
151 Unimplemented();
152 }
153
154 void ShenandoahBarrierSet::write_prim_array(MemRegion mr) {
155 Unimplemented();
156 }
157
158 void ShenandoahBarrierSet::write_prim_field(HeapWord* hw, size_t s , juint x, juint y) {
159 Unimplemented();
160 }
161
162 bool ShenandoahBarrierSet::write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y) {
163 Unimplemented();
164 return false;
165 }
166
167 bool ShenandoahBarrierSet::need_update_refs_barrier() {
168 if (UseShenandoahMatrix) {
169 return true;
170 }
171 if (_heap->shenandoahPolicy()->update_refs()) {
172 return _heap->is_update_refs_in_progress();
173 } else {
174 return _heap->concurrent_mark_in_progress() && _heap->need_update_refs();
175 }
176 }
177
178 void ShenandoahBarrierSet::write_ref_array_work(MemRegion r) {
179 ShouldNotReachHere();
180 }
181
182 template <class T, bool UPDATE_MATRIX, bool STOREVAL_WRITE_BARRIER>
183 void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) {
184 ShenandoahUpdateRefsForOopClosure<UPDATE_MATRIX, STOREVAL_WRITE_BARRIER> cl;
185 T* dst = (T*) start;
186 for (size_t i = 0; i < count; i++) {
187 cl.do_oop(dst++);
188 }
189 }
190
191 void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) {
192 if (! need_update_refs_barrier()) return;
193 ShenandoahHeap* heap = ShenandoahHeap::heap();
194 ShenandoahConnectionMatrix* matrix = heap->connection_matrix();
195 if (UseCompressedOops) {
196 if (UseShenandoahMatrix) {
197 if (_heap->is_concurrent_partial_in_progress()) {
198 write_ref_array_loop<narrowOop, true, true>(start, count);
199 } else {
200 write_ref_array_loop<narrowOop, true, false>(start, count);
201 }
202 } else {
203 assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix");
204 write_ref_array_loop<narrowOop, false, false>(start, count);
205 }
206 } else {
207 if (UseShenandoahMatrix) {
208 if (_heap->is_concurrent_partial_in_progress()) {
209 write_ref_array_loop<oop, true, true>(start, count);
210 } else {
211 write_ref_array_loop<oop, true, false>(start, count);
212 }
213 } else {
214 assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix");
215 write_ref_array_loop<oop, false, false>(start, count);
216 }
217 }
218 }
219
220 template <class T>
221 void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, int count) {
222
223 #ifdef ASSERT
224 if (_heap->is_in(dst) &&
225 _heap->in_collection_set(dst) &&
226 ! _heap->cancelled_concgc()) {
227 tty->print_cr("dst = "PTR_FORMAT, p2i(dst));
228 _heap->heap_region_containing((HeapWord*) dst)->print();
229 assert(false, "We should have fixed this earlier");
230 }
231 #endif
232
233 if (ShenandoahSATBBarrier ||
234 (ShenandoahConditionalSATBBarrier && _heap->concurrent_mark_in_progress())) {
235 T* elem_ptr = dst;
236 for (int i = 0; i < count; i++, elem_ptr++) {
237 T heap_oop = oopDesc::load_heap_oop(elem_ptr);
238 if (!oopDesc::is_null(heap_oop)) {
239 G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop_not_null(heap_oop));
240 }
241 }
242 }
243 }
244
245 void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) {
246 if (! dest_uninitialized) {
247 write_ref_array_pre_work(dst, count);
248 }
249 }
250
251 void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) {
252 if (! dest_uninitialized) {
253 write_ref_array_pre_work(dst, count);
254 }
255 }
256
257 template <class T>
258 inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop new_val) {
259 #ifdef ASSERT
260 {
261 if (_heap->is_in(field) &&
262 _heap->in_collection_set(field) &&
263 ! _heap->cancelled_concgc()) {
264 tty->print_cr("field = "PTR_FORMAT, p2i(field));
265 tty->print_cr("in_cset: %s", BOOL_TO_STR(_heap->in_collection_set(field)));
266 _heap->heap_region_containing((HeapWord*)field)->print();
267 tty->print_cr("marking: %s, evacuating: %s",
268 BOOL_TO_STR(_heap->concurrent_mark_in_progress()),
269 BOOL_TO_STR(_heap->is_evacuation_in_progress()));
270 assert(false, "We should have fixed this earlier");
271 }
272 }
273 #endif
274
275 if (_heap->concurrent_mark_in_progress()) {
276 T heap_oop = oopDesc::load_heap_oop(field);
277 if (!oopDesc::is_null(heap_oop)) {
278 G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop(heap_oop));
279 }
280 }
281 if (UseShenandoahMatrix && ! oopDesc::is_null(new_val)) {
282 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
283 matrix->set_connected(field, new_val);
284 }
285 }
286
287 // These are the more general virtual versions.
288 void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) {
289 inline_write_ref_field_pre(field, new_val);
290 }
291
292 void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) {
293 inline_write_ref_field_pre(field, new_val);
294 }
295
296 void ShenandoahBarrierSet::write_ref_field_pre_work(void* field, oop new_val) {
297 guarantee(false, "Not needed");
298 }
299
300 void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) {
301 #ifdef ASSERT
302 ShenandoahHeap* heap = ShenandoahHeap::heap();
303 if (!(heap->cancelled_concgc() || !heap->in_collection_set(v))) {
304 tty->print_cr("field not in collection set: "PTR_FORMAT, p2i(v));
305 tty->print_cr("containing heap region:");
306 ShenandoahHeap::heap()->heap_region_containing(v)->print();
307 }
308 assert(heap->cancelled_concgc() || !heap->in_collection_set(v), "only write to to-space");
309 if (_heap->concurrent_mark_in_progress()) {
310 assert(o == NULL || oopDesc::unsafe_equals(o, resolve_oop_static(o)), "only write to-space values");
311 assert(o == NULL || !heap->in_collection_set(o), "only write to-space values");
312 }
313 #endif
314 }
315
316 void ShenandoahBarrierSet::write_region_work(MemRegion mr) {
317
318 if (! need_update_refs_barrier()) return;
319
320 // This is called for cloning an object (see jvm.cpp) after the clone
321 // has been made. We are not interested in any 'previous value' because
322 // it would be NULL in any case. But we *are* interested in any oop*
323 // that potentially need to be updated.
324
325 oop obj = oop(mr.start());
326 assert(oopDesc::is_oop(obj), "must be an oop");
327 if (UseShenandoahMatrix) {
328 if (_heap->is_concurrent_partial_in_progress()) {
329 ShenandoahUpdateRefsForOopClosure<true, true> cl;
330 obj->oop_iterate(&cl);
331 } else {
332 ShenandoahUpdateRefsForOopClosure<true, false> cl;
333 obj->oop_iterate(&cl);
334 }
335 } else {
336 assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix");
337 ShenandoahUpdateRefsForOopClosure<false, false> cl;
338 obj->oop_iterate(&cl);
339 }
340 }
341
342 oop ShenandoahBarrierSet::read_barrier(oop src) {
343 if (ShenandoahReadBarrier) {
344 return ShenandoahBarrierSet::resolve_oop_static(src);
345 } else {
346 return src;
347 }
348 }
349
350 bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) {
351 bool eq = oopDesc::unsafe_equals(obj1, obj2);
352 if (! eq && ShenandoahAcmpBarrier) {
353 OrderAccess::loadload();
354 obj1 = resolve_oop_static(obj1);
355 obj2 = resolve_oop_static(obj2);
356 eq = oopDesc::unsafe_equals(obj1, obj2);
357 }
358 return eq;
359 }
360
361 bool ShenandoahBarrierSet::obj_equals(narrowOop obj1, narrowOop obj2) {
362 return obj_equals(oopDesc::decode_heap_oop(obj1), oopDesc::decode_heap_oop(obj2));
363 }
364
365 #ifdef ASSERT
366 bool ShenandoahBarrierSet::is_safe(oop o) {
367 if (o == NULL) return true;
368 if (_heap->in_collection_set(o)) {
369 return false;
370 }
371 if (! oopDesc::unsafe_equals(o, read_barrier(o))) {
372 return false;
373 }
374 return true;
375 }
376
377 bool ShenandoahBarrierSet::is_safe(narrowOop o) {
378 return is_safe(oopDesc::decode_heap_oop(o));
379 }
380 #endif
381
382 JRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_JRT(oopDesc* src))
383 oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->write_barrier(src);
384 return (oopDesc*) result;
385 JRT_END
386
387 IRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_IRT(oopDesc* src))
388 oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->write_barrier(src);
389 return (oopDesc*) result;
390 IRT_END
391
392 oop ShenandoahBarrierSet::write_barrier_impl(oop obj) {
393 if (!oopDesc::is_null(obj)) {
394 bool evac_in_progress = _heap->is_evacuation_in_progress();
395 OrderAccess::loadload();
396 oop fwd = resolve_oop_static_not_null(obj);
397 if (evac_in_progress &&
398 _heap->in_collection_set(obj) &&
399 oopDesc::unsafe_equals(obj, fwd)) {
400 bool evac;
401 oop copy = _heap->evacuate_object(obj, Thread::current(), evac);
402 if (evac && _heap->is_concurrent_partial_in_progress()) {
403 G1SATBCardTableModRefBS::enqueue(copy);
404 }
405 return copy;
406 } else {
407 return fwd;
408 }
409 } else {
410 return obj;
411 }
412 }
413
414 oop ShenandoahBarrierSet::write_barrier(oop obj) {
415 if (ShenandoahWriteBarrier) {
416 return write_barrier_impl(obj);
417 } else {
418 return obj;
419 }
420 }
421
422 oop ShenandoahBarrierSet::storeval_barrier(oop obj) {
423 if (ShenandoahStoreValWriteBarrier) {
424 return write_barrier_impl(obj);
425 }
426 if (ShenandoahStoreValReadBarrier) {
427 return resolve_oop_static(obj);
428 }
429 return obj;
430 }
431
432 void ShenandoahBarrierSet::keep_alive_barrier(oop obj) {
433 if (ShenandoahKeepAliveBarrier) {
434 if (_heap->concurrent_mark_in_progress()) {
435 G1SATBCardTableModRefBS::enqueue(obj);
436 } else if (_heap->is_concurrent_partial_in_progress()) {
437 write_barrier_impl(obj);
438 }
439 }
440 }
441
442 #ifdef ASSERT
443 void ShenandoahBarrierSet::verify_safe_oop(oop p) {
444 ShenandoahHeap* heap = ShenandoahHeap::heap();
445 if (p == NULL) return;
446 if (heap->in_collection_set(p) &&
447 ! heap->cancelled_concgc()) {
448 tty->print_cr("oop = "PTR_FORMAT", resolved: "PTR_FORMAT", marked-next %s, marked-complete: %s",
449 p2i(p),
450 p2i(read_barrier(p)),
451 BOOL_TO_STR(heap->is_marked_next(p)),
452 BOOL_TO_STR(heap->is_marked_complete(p)));
453 tty->print_cr("in_cset: %s", BOOL_TO_STR(heap->in_collection_set(p)));
454 heap->heap_region_containing((HeapWord*) p)->print();
455 tty->print_cr("top-at-mark-start: %p", heap->next_top_at_mark_start((HeapWord*) p));
456 tty->print_cr("top-at-prev-mark-start: %p", heap->complete_top_at_mark_start((HeapWord*) p));
457 tty->print_cr("marking: %s, evacuating: %s", BOOL_TO_STR(heap->concurrent_mark_in_progress()), BOOL_TO_STR(heap->is_evacuation_in_progress()));
458 assert(false, "We should have fixed this earlier");
459 }
460 }
461 #endif