1 /* 2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/g1/g1SATBCardTableModRefBS.hpp" 26 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 27 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 28 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp" 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 30 #include "runtime/interfaceSupport.hpp" 31 32 template <bool UPDATE_MATRIX, bool STOREVAL_WRITE_BARRIER> 33 class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure { 34 private: 35 ShenandoahHeap* _heap; 36 template <class T> 37 inline void do_oop_work(T* p) { 38 oop o; 39 if (STOREVAL_WRITE_BARRIER) { 40 bool evac; 41 o = _heap->evac_update_oop_ref(p, evac); 42 if (evac) { 43 G1SATBCardTableModRefBS::enqueue(o); 44 } 45 } else { 46 o = _heap->maybe_update_oop_ref(p); 47 } 48 if (UPDATE_MATRIX && !oopDesc::is_null(o)) { 49 _heap->connection_matrix()->set_connected(p, o); 50 } 51 } 52 public: 53 ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()) {} 54 void do_oop(oop* p) { do_oop_work(p); } 55 void do_oop(narrowOop* p) { do_oop_work(p); } 56 }; 57 58 ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : 59 BarrierSet(BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)), 60 _heap(heap) 61 { 62 } 63 64 void ShenandoahBarrierSet::print_on(outputStream* st) const { 65 st->print("ShenandoahBarrierSet"); 66 } 67 68 bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) { 69 return bsn == BarrierSet::ShenandoahBarrierSet; 70 } 71 72 bool ShenandoahBarrierSet::has_read_prim_array_opt() { 73 return true; 74 } 75 76 bool ShenandoahBarrierSet::has_read_prim_barrier() { 77 return false; 78 } 79 80 bool ShenandoahBarrierSet::has_read_ref_array_opt() { 81 return true; 82 } 83 84 bool ShenandoahBarrierSet::has_read_ref_barrier() { 85 return false; 86 } 87 88 bool ShenandoahBarrierSet::has_read_region_opt() { 89 return true; 90 } 91 92 bool ShenandoahBarrierSet::has_write_prim_array_opt() { 93 return true; 94 } 95 96 bool ShenandoahBarrierSet::has_write_prim_barrier() { 97 return false; 98 } 99 100 bool ShenandoahBarrierSet::has_write_ref_array_opt() { 101 return true; 102 } 103 104 bool ShenandoahBarrierSet::has_write_ref_barrier() { 105 return true; 106 } 107 108 bool ShenandoahBarrierSet::has_write_ref_pre_barrier() { 109 return true; 110 } 111 112 bool ShenandoahBarrierSet::has_write_region_opt() { 113 return true; 114 } 115 116 bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) { 117 return true; 118 } 119 120 void ShenandoahBarrierSet::read_prim_array(MemRegion mr) { 121 Unimplemented(); 122 } 123 124 void ShenandoahBarrierSet::read_prim_field(HeapWord* hw, size_t s){ 125 Unimplemented(); 126 } 127 128 bool ShenandoahBarrierSet::read_prim_needs_barrier(HeapWord* hw, size_t s) { 129 return false; 130 } 131 132 void ShenandoahBarrierSet::read_ref_array(MemRegion mr) { 133 Unimplemented(); 134 } 135 136 void ShenandoahBarrierSet::read_ref_field(void* v) { 137 // tty->print_cr("read_ref_field: v = "PTR_FORMAT, v); 138 // return *v; 139 } 140 141 bool ShenandoahBarrierSet::read_ref_needs_barrier(void* v) { 142 Unimplemented(); 143 return false; 144 } 145 146 void ShenandoahBarrierSet::read_region(MemRegion mr) { 147 Unimplemented(); 148 } 149 150 void ShenandoahBarrierSet::resize_covered_region(MemRegion mr) { 151 Unimplemented(); 152 } 153 154 void ShenandoahBarrierSet::write_prim_array(MemRegion mr) { 155 Unimplemented(); 156 } 157 158 void ShenandoahBarrierSet::write_prim_field(HeapWord* hw, size_t s , juint x, juint y) { 159 Unimplemented(); 160 } 161 162 bool ShenandoahBarrierSet::write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y) { 163 Unimplemented(); 164 return false; 165 } 166 167 bool ShenandoahBarrierSet::need_update_refs_barrier() { 168 if (UseShenandoahMatrix) { 169 return true; 170 } 171 if (_heap->shenandoahPolicy()->update_refs()) { 172 return _heap->is_update_refs_in_progress(); 173 } else { 174 return _heap->concurrent_mark_in_progress() && _heap->need_update_refs(); 175 } 176 } 177 178 void ShenandoahBarrierSet::write_ref_array_work(MemRegion r) { 179 ShouldNotReachHere(); 180 } 181 182 template <class T, bool UPDATE_MATRIX, bool STOREVAL_WRITE_BARRIER> 183 void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) { 184 ShenandoahUpdateRefsForOopClosure<UPDATE_MATRIX, STOREVAL_WRITE_BARRIER> cl; 185 T* dst = (T*) start; 186 for (size_t i = 0; i < count; i++) { 187 cl.do_oop(dst++); 188 } 189 } 190 191 void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) { 192 if (! need_update_refs_barrier()) return; 193 ShenandoahHeap* heap = ShenandoahHeap::heap(); 194 ShenandoahConnectionMatrix* matrix = heap->connection_matrix(); 195 if (UseCompressedOops) { 196 if (UseShenandoahMatrix) { 197 if (_heap->is_concurrent_partial_in_progress()) { 198 write_ref_array_loop<narrowOop, true, true>(start, count); 199 } else { 200 write_ref_array_loop<narrowOop, true, false>(start, count); 201 } 202 } else { 203 assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix"); 204 write_ref_array_loop<narrowOop, false, false>(start, count); 205 } 206 } else { 207 if (UseShenandoahMatrix) { 208 if (_heap->is_concurrent_partial_in_progress()) { 209 write_ref_array_loop<oop, true, true>(start, count); 210 } else { 211 write_ref_array_loop<oop, true, false>(start, count); 212 } 213 } else { 214 assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix"); 215 write_ref_array_loop<oop, false, false>(start, count); 216 } 217 } 218 } 219 220 template <class T> 221 void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, int count) { 222 223 #ifdef ASSERT 224 if (_heap->is_in(dst) && 225 _heap->in_collection_set(dst) && 226 ! _heap->cancelled_concgc()) { 227 tty->print_cr("dst = "PTR_FORMAT, p2i(dst)); 228 _heap->heap_region_containing((HeapWord*) dst)->print(); 229 assert(false, "We should have fixed this earlier"); 230 } 231 #endif 232 233 if (ShenandoahSATBBarrier || 234 (ShenandoahConditionalSATBBarrier && _heap->concurrent_mark_in_progress())) { 235 T* elem_ptr = dst; 236 for (int i = 0; i < count; i++, elem_ptr++) { 237 T heap_oop = oopDesc::load_heap_oop(elem_ptr); 238 if (!oopDesc::is_null(heap_oop)) { 239 G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop_not_null(heap_oop)); 240 } 241 } 242 } 243 } 244 245 void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) { 246 if (! dest_uninitialized) { 247 write_ref_array_pre_work(dst, count); 248 } 249 } 250 251 void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) { 252 if (! dest_uninitialized) { 253 write_ref_array_pre_work(dst, count); 254 } 255 } 256 257 template <class T> 258 inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop new_val) { 259 #ifdef ASSERT 260 { 261 if (_heap->is_in(field) && 262 _heap->in_collection_set(field) && 263 ! _heap->cancelled_concgc()) { 264 tty->print_cr("field = "PTR_FORMAT, p2i(field)); 265 tty->print_cr("in_cset: %s", BOOL_TO_STR(_heap->in_collection_set(field))); 266 _heap->heap_region_containing((HeapWord*)field)->print(); 267 tty->print_cr("marking: %s, evacuating: %s", 268 BOOL_TO_STR(_heap->concurrent_mark_in_progress()), 269 BOOL_TO_STR(_heap->is_evacuation_in_progress())); 270 assert(false, "We should have fixed this earlier"); 271 } 272 } 273 #endif 274 275 if (_heap->concurrent_mark_in_progress()) { 276 T heap_oop = oopDesc::load_heap_oop(field); 277 if (!oopDesc::is_null(heap_oop)) { 278 G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop(heap_oop)); 279 } 280 } 281 if (UseShenandoahMatrix && ! oopDesc::is_null(new_val)) { 282 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix(); 283 matrix->set_connected(field, new_val); 284 } 285 } 286 287 // These are the more general virtual versions. 288 void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) { 289 inline_write_ref_field_pre(field, new_val); 290 } 291 292 void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) { 293 inline_write_ref_field_pre(field, new_val); 294 } 295 296 void ShenandoahBarrierSet::write_ref_field_pre_work(void* field, oop new_val) { 297 guarantee(false, "Not needed"); 298 } 299 300 void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) { 301 #ifdef ASSERT 302 ShenandoahHeap* heap = ShenandoahHeap::heap(); 303 if (!(heap->cancelled_concgc() || !heap->in_collection_set(v))) { 304 tty->print_cr("field not in collection set: "PTR_FORMAT, p2i(v)); 305 tty->print_cr("containing heap region:"); 306 ShenandoahHeap::heap()->heap_region_containing(v)->print(); 307 } 308 assert(heap->cancelled_concgc() || !heap->in_collection_set(v), "only write to to-space"); 309 if (_heap->concurrent_mark_in_progress()) { 310 assert(o == NULL || oopDesc::unsafe_equals(o, resolve_oop_static(o)), "only write to-space values"); 311 assert(o == NULL || !heap->in_collection_set(o), "only write to-space values"); 312 } 313 #endif 314 } 315 316 void ShenandoahBarrierSet::write_region_work(MemRegion mr) { 317 318 if (! need_update_refs_barrier()) return; 319 320 // This is called for cloning an object (see jvm.cpp) after the clone 321 // has been made. We are not interested in any 'previous value' because 322 // it would be NULL in any case. But we *are* interested in any oop* 323 // that potentially need to be updated. 324 325 oop obj = oop(mr.start()); 326 assert(oopDesc::is_oop(obj), "must be an oop"); 327 if (UseShenandoahMatrix) { 328 if (_heap->is_concurrent_partial_in_progress()) { 329 ShenandoahUpdateRefsForOopClosure<true, true> cl; 330 obj->oop_iterate(&cl); 331 } else { 332 ShenandoahUpdateRefsForOopClosure<true, false> cl; 333 obj->oop_iterate(&cl); 334 } 335 } else { 336 assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix"); 337 ShenandoahUpdateRefsForOopClosure<false, false> cl; 338 obj->oop_iterate(&cl); 339 } 340 } 341 342 oop ShenandoahBarrierSet::read_barrier(oop src) { 343 if (ShenandoahReadBarrier) { 344 return ShenandoahBarrierSet::resolve_oop_static(src); 345 } else { 346 return src; 347 } 348 } 349 350 bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) { 351 bool eq = oopDesc::unsafe_equals(obj1, obj2); 352 if (! eq && ShenandoahAcmpBarrier) { 353 OrderAccess::loadload(); 354 obj1 = resolve_oop_static(obj1); 355 obj2 = resolve_oop_static(obj2); 356 eq = oopDesc::unsafe_equals(obj1, obj2); 357 } 358 return eq; 359 } 360 361 bool ShenandoahBarrierSet::obj_equals(narrowOop obj1, narrowOop obj2) { 362 return obj_equals(oopDesc::decode_heap_oop(obj1), oopDesc::decode_heap_oop(obj2)); 363 } 364 365 #ifdef ASSERT 366 bool ShenandoahBarrierSet::is_safe(oop o) { 367 if (o == NULL) return true; 368 if (_heap->in_collection_set(o)) { 369 return false; 370 } 371 if (! oopDesc::unsafe_equals(o, read_barrier(o))) { 372 return false; 373 } 374 return true; 375 } 376 377 bool ShenandoahBarrierSet::is_safe(narrowOop o) { 378 return is_safe(oopDesc::decode_heap_oop(o)); 379 } 380 #endif 381 382 JRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_JRT(oopDesc* src)) 383 oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->write_barrier(src); 384 return (oopDesc*) result; 385 JRT_END 386 387 IRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_IRT(oopDesc* src)) 388 oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->write_barrier(src); 389 return (oopDesc*) result; 390 IRT_END 391 392 oop ShenandoahBarrierSet::write_barrier_impl(oop obj) { 393 if (!oopDesc::is_null(obj)) { 394 bool evac_in_progress = _heap->is_evacuation_in_progress(); 395 OrderAccess::loadload(); 396 oop fwd = resolve_oop_static_not_null(obj); 397 if (evac_in_progress && 398 _heap->in_collection_set(obj) && 399 oopDesc::unsafe_equals(obj, fwd)) { 400 bool evac; 401 oop copy = _heap->evacuate_object(obj, Thread::current(), evac); 402 if (evac && _heap->is_concurrent_partial_in_progress()) { 403 G1SATBCardTableModRefBS::enqueue(copy); 404 } 405 return copy; 406 } else { 407 return fwd; 408 } 409 } else { 410 return obj; 411 } 412 } 413 414 oop ShenandoahBarrierSet::write_barrier(oop obj) { 415 if (ShenandoahWriteBarrier) { 416 return write_barrier_impl(obj); 417 } else { 418 return obj; 419 } 420 } 421 422 oop ShenandoahBarrierSet::storeval_barrier(oop obj) { 423 if (ShenandoahStoreValWriteBarrier) { 424 return write_barrier_impl(obj); 425 } 426 if (ShenandoahStoreValReadBarrier) { 427 return resolve_oop_static(obj); 428 } 429 return obj; 430 } 431 432 void ShenandoahBarrierSet::keep_alive_barrier(oop obj) { 433 if (ShenandoahKeepAliveBarrier) { 434 if (_heap->concurrent_mark_in_progress()) { 435 G1SATBCardTableModRefBS::enqueue(obj); 436 } else if (_heap->is_concurrent_partial_in_progress()) { 437 write_barrier_impl(obj); 438 } 439 } 440 } 441 442 #ifdef ASSERT 443 void ShenandoahBarrierSet::verify_safe_oop(oop p) { 444 ShenandoahHeap* heap = ShenandoahHeap::heap(); 445 if (p == NULL) return; 446 if (heap->in_collection_set(p) && 447 ! heap->cancelled_concgc()) { 448 tty->print_cr("oop = "PTR_FORMAT", resolved: "PTR_FORMAT", marked %s", 449 p2i(p), 450 p2i(read_barrier(p)), 451 BOOL_TO_STR(heap->is_marked(p))); 452 tty->print_cr("in_cset: %s", BOOL_TO_STR(heap->in_collection_set(p))); 453 heap->heap_region_containing((HeapWord*) p)->print(); 454 tty->print_cr("top-at-mark-start: %p", heap->top_at_mark_start((HeapWord*) p)); 455 tty->print_cr("marking: %s, evacuating: %s", BOOL_TO_STR(heap->concurrent_mark_in_progress()), BOOL_TO_STR(heap->is_evacuation_in_progress())); 456 assert(false, "We should have fixed this earlier"); 457 } 458 } 459 #endif