1 /* 2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/g1/g1SATBCardTableModRefBS.hpp" 26 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 27 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 28 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp" 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 30 #include "runtime/interfaceSupport.hpp" 31 32 class UpdateRefsForOopClosure: public ExtendedOopClosure { 33 34 private: 35 ShenandoahHeap* _heap; 36 template <class T> 37 inline void do_oop_work(T* p) { 38 _heap->maybe_update_oop_ref(p); 39 } 40 public: 41 UpdateRefsForOopClosure() { 42 _heap = ShenandoahHeap::heap(); 43 } 44 45 void do_oop(oop* p) { 46 do_oop_work(p); 47 } 48 49 void do_oop(narrowOop* p) { 50 do_oop_work(p); 51 } 52 53 }; 54 55 class UpdateRefsForOopMatrixClosure: public ExtendedOopClosure { 56 57 private: 58 ShenandoahHeap* _heap; 59 template <class T> 60 inline void do_oop_work(T* p) { 61 oop o = _heap->maybe_update_oop_ref(p); 62 if (! oopDesc::is_null(o)) { 63 _heap->connection_matrix()->set_connected(p, o); 64 } 65 } 66 public: 67 UpdateRefsForOopMatrixClosure() { 68 _heap = ShenandoahHeap::heap(); 69 } 70 71 void do_oop(oop* p) { 72 do_oop_work(p); 73 } 74 75 void do_oop(narrowOop* p) { 76 do_oop_work(p); 77 } 78 79 }; 80 81 ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : 82 BarrierSet(BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)), 83 _heap(heap) 84 { 85 } 86 87 void ShenandoahBarrierSet::print_on(outputStream* st) const { 88 st->print("ShenandoahBarrierSet"); 89 } 90 91 bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) { 92 return bsn == BarrierSet::ShenandoahBarrierSet; 93 } 94 95 bool ShenandoahBarrierSet::has_read_prim_array_opt() { 96 return true; 97 } 98 99 bool ShenandoahBarrierSet::has_read_prim_barrier() { 100 return false; 101 } 102 103 bool ShenandoahBarrierSet::has_read_ref_array_opt() { 104 return true; 105 } 106 107 bool ShenandoahBarrierSet::has_read_ref_barrier() { 108 return false; 109 } 110 111 bool ShenandoahBarrierSet::has_read_region_opt() { 112 return true; 113 } 114 115 bool ShenandoahBarrierSet::has_write_prim_array_opt() { 116 return true; 117 } 118 119 bool ShenandoahBarrierSet::has_write_prim_barrier() { 120 return false; 121 } 122 123 bool ShenandoahBarrierSet::has_write_ref_array_opt() { 124 return true; 125 } 126 127 bool ShenandoahBarrierSet::has_write_ref_barrier() { 128 return true; 129 } 130 131 bool ShenandoahBarrierSet::has_write_ref_pre_barrier() { 132 return true; 133 } 134 135 bool ShenandoahBarrierSet::has_write_region_opt() { 136 return true; 137 } 138 139 bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) { 140 return true; 141 } 142 143 void ShenandoahBarrierSet::read_prim_array(MemRegion mr) { 144 Unimplemented(); 145 } 146 147 void ShenandoahBarrierSet::read_prim_field(HeapWord* hw, size_t s){ 148 Unimplemented(); 149 } 150 151 bool ShenandoahBarrierSet::read_prim_needs_barrier(HeapWord* hw, size_t s) { 152 return false; 153 } 154 155 void ShenandoahBarrierSet::read_ref_array(MemRegion mr) { 156 Unimplemented(); 157 } 158 159 void ShenandoahBarrierSet::read_ref_field(void* v) { 160 // tty->print_cr("read_ref_field: v = "PTR_FORMAT, v); 161 // return *v; 162 } 163 164 bool ShenandoahBarrierSet::read_ref_needs_barrier(void* v) { 165 Unimplemented(); 166 return false; 167 } 168 169 void ShenandoahBarrierSet::read_region(MemRegion mr) { 170 Unimplemented(); 171 } 172 173 void ShenandoahBarrierSet::resize_covered_region(MemRegion mr) { 174 Unimplemented(); 175 } 176 177 void ShenandoahBarrierSet::write_prim_array(MemRegion mr) { 178 Unimplemented(); 179 } 180 181 void ShenandoahBarrierSet::write_prim_field(HeapWord* hw, size_t s , juint x, juint y) { 182 Unimplemented(); 183 } 184 185 bool ShenandoahBarrierSet::write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y) { 186 Unimplemented(); 187 return false; 188 } 189 190 bool ShenandoahBarrierSet::need_update_refs_barrier() { 191 if (UseShenandoahMatrix) { 192 return true; 193 } 194 if (_heap->shenandoahPolicy()->update_refs_early()) { 195 return _heap->is_update_refs_in_progress(); 196 } else { 197 return _heap->concurrent_mark_in_progress() && _heap->need_update_refs(); 198 } 199 } 200 201 void ShenandoahBarrierSet::write_ref_array_work(MemRegion r) { 202 ShouldNotReachHere(); 203 } 204 205 void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) { 206 if (! need_update_refs_barrier()) return; 207 ShenandoahHeap* heap = ShenandoahHeap::heap(); 208 ShenandoahConnectionMatrix* matrix = heap->connection_matrix(); 209 // TODO: Use templated loop and split on oop/narrowOop and UseShenandoahMatrix. 210 if (UseCompressedOops) { 211 narrowOop* dst = (narrowOop*) start; 212 for (size_t i = 0; i < count; i++, dst++) { 213 oop o = _heap->maybe_update_oop_ref(dst); 214 if (UseShenandoahMatrix && ! oopDesc::is_null(o)) { 215 matrix->set_connected(dst, o); 216 } 217 } 218 } else { 219 oop* dst = (oop*) start; 220 for (size_t i = 0; i < count; i++, dst++) { 221 oop o = _heap->maybe_update_oop_ref(dst); 222 if (UseShenandoahMatrix && ! oopDesc::is_null(o)) { 223 matrix->set_connected(dst, o); 224 } 225 } 226 } 227 } 228 229 template <class T> 230 void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, int count) { 231 232 #ifdef ASSERT 233 if (_heap->is_in(dst) && 234 _heap->in_collection_set(dst) && 235 ! _heap->cancelled_concgc()) { 236 tty->print_cr("dst = "PTR_FORMAT, p2i(dst)); 237 _heap->heap_region_containing((HeapWord*) dst)->print(); 238 assert(false, "We should have fixed this earlier"); 239 } 240 #endif 241 242 if (! JavaThread::satb_mark_queue_set().is_active()) return; 243 T* elem_ptr = dst; 244 for (int i = 0; i < count; i++, elem_ptr++) { 245 T heap_oop = oopDesc::load_heap_oop(elem_ptr); 246 if (!oopDesc::is_null(heap_oop)) { 247 G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop_not_null(heap_oop)); 248 } 249 } 250 } 251 252 void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) { 253 if (! dest_uninitialized) { 254 write_ref_array_pre_work(dst, count); 255 } 256 } 257 258 void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) { 259 if (! dest_uninitialized) { 260 write_ref_array_pre_work(dst, count); 261 } 262 } 263 264 template <class T> 265 void ShenandoahBarrierSet::write_ref_field_pre_static(T* field, oop new_val) { 266 T heap_oop = oopDesc::load_heap_oop(field); 267 268 #ifdef ASSERT 269 { 270 ShenandoahHeap* heap = ShenandoahHeap::heap(); 271 if (heap->is_in(field) && 272 heap->in_collection_set(field) && 273 ! heap->cancelled_concgc()) { 274 tty->print_cr("field = "PTR_FORMAT, p2i(field)); 275 tty->print_cr("in_cset: %s", BOOL_TO_STR(heap->in_collection_set(field))); 276 heap->heap_region_containing((HeapWord*)field)->print(); 277 tty->print_cr("marking: %s, evacuating: %s", 278 BOOL_TO_STR(heap->concurrent_mark_in_progress()), 279 BOOL_TO_STR(heap->is_evacuation_in_progress())); 280 assert(false, "We should have fixed this earlier"); 281 } 282 } 283 #endif 284 285 if (!oopDesc::is_null(heap_oop)) { 286 G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop(heap_oop)); 287 } 288 if (UseShenandoahMatrix && ! oopDesc::is_null(new_val)) { 289 ShenandoahHeap* heap = ShenandoahHeap::heap(); 290 ShenandoahConnectionMatrix* matrix = heap->connection_matrix(); 291 matrix->set_connected(field, new_val); 292 } 293 } 294 295 template <class T> 296 inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) { 297 write_ref_field_pre_static(field, newVal); 298 } 299 300 // These are the more general virtual versions. 301 void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) { 302 write_ref_field_pre_static(field, new_val); 303 } 304 305 void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) { 306 write_ref_field_pre_static(field, new_val); 307 } 308 309 void ShenandoahBarrierSet::write_ref_field_pre_work(void* field, oop new_val) { 310 guarantee(false, "Not needed"); 311 } 312 313 void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) { 314 #ifdef ASSERT 315 ShenandoahHeap* heap = ShenandoahHeap::heap(); 316 if (!(heap->cancelled_concgc() || !heap->in_collection_set(v))) { 317 tty->print_cr("field not in collection set: "PTR_FORMAT, p2i(v)); 318 tty->print_cr("containing heap region:"); 319 ShenandoahHeap::heap()->heap_region_containing(v)->print(); 320 } 321 assert(heap->cancelled_concgc() || !heap->in_collection_set(v), "only write to to-space"); 322 if (_heap->concurrent_mark_in_progress()) { 323 assert(o == NULL || oopDesc::unsafe_equals(o, resolve_oop_static(o)), "only write to-space values"); 324 assert(o == NULL || !heap->in_collection_set(o), "only write to-space values"); 325 } 326 #endif 327 if (UseShenandoahMatrix && ! oopDesc::is_null(o)) { 328 ShenandoahHeap* heap = ShenandoahHeap::heap(); 329 ShenandoahConnectionMatrix* matrix = heap->connection_matrix(); 330 matrix->set_connected(v, o); 331 } 332 } 333 334 void ShenandoahBarrierSet::write_region_work(MemRegion mr) { 335 336 if (! need_update_refs_barrier()) return; 337 338 // This is called for cloning an object (see jvm.cpp) after the clone 339 // has been made. We are not interested in any 'previous value' because 340 // it would be NULL in any case. But we *are* interested in any oop* 341 // that potentially need to be updated. 342 343 oop obj = oop(mr.start()); 344 assert(obj->is_oop(), "must be an oop"); 345 if (UseShenandoahMatrix) { 346 UpdateRefsForOopMatrixClosure cl; 347 obj->oop_iterate(&cl); 348 } else { 349 UpdateRefsForOopClosure cl; 350 obj->oop_iterate(&cl); 351 } 352 } 353 354 oop ShenandoahBarrierSet::read_barrier(oop src) { 355 if (ShenandoahReadBarrier) { 356 return ShenandoahBarrierSet::resolve_oop_static(src); 357 } else { 358 return src; 359 } 360 } 361 362 bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) { 363 bool eq = oopDesc::unsafe_equals(obj1, obj2); 364 if (! eq && ShenandoahAcmpBarrier) { 365 OrderAccess::loadload(); 366 obj1 = resolve_oop_static(obj1); 367 obj2 = resolve_oop_static(obj2); 368 eq = oopDesc::unsafe_equals(obj1, obj2); 369 } 370 return eq; 371 } 372 373 bool ShenandoahBarrierSet::obj_equals(narrowOop obj1, narrowOop obj2) { 374 return obj_equals(oopDesc::decode_heap_oop(obj1), oopDesc::decode_heap_oop(obj2)); 375 } 376 377 #ifdef ASSERT 378 bool ShenandoahBarrierSet::is_safe(oop o) { 379 if (o == NULL) return true; 380 if (_heap->in_collection_set(o)) { 381 return false; 382 } 383 if (! oopDesc::unsafe_equals(o, read_barrier(o))) { 384 return false; 385 } 386 return true; 387 } 388 389 bool ShenandoahBarrierSet::is_safe(narrowOop o) { 390 return is_safe(oopDesc::decode_heap_oop(o)); 391 } 392 #endif 393 394 oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop_work(oop src) { 395 assert(src != NULL, "only evacuated non NULL oops"); 396 397 if (_heap->in_collection_set(src)) { 398 return resolve_and_maybe_copy_oop_work2(src); 399 } else { 400 return src; 401 } 402 } 403 404 oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop_work2(oop src) { 405 assert(src != NULL, "only evacuated non NULL oops"); 406 assert(_heap->in_collection_set(src), "only evacuate objects in collection set"); 407 assert(! _heap->heap_region_containing(src)->is_humongous(), "never evacuate humongous objects"); 408 // TODO: Consider passing thread from caller. 409 bool evac; 410 oop dst = _heap->evacuate_object(src, Thread::current(), evac); 411 412 log_develop_trace(gc, compaction)("src = "PTR_FORMAT" dst = "PTR_FORMAT" src-2 = "PTR_FORMAT, 413 p2i(src), p2i(dst), p2i(((HeapWord*) src) - 2)); 414 415 assert(_heap->is_in(dst), "result should be in the heap"); 416 return dst; 417 } 418 419 oop ShenandoahBarrierSet::resolve_and_maybe_copy_oopHelper(oop src) { 420 assert(src != NULL, "checked before"); 421 bool evac = _heap->is_evacuation_in_progress(); 422 OrderAccess::loadload(); 423 src = resolve_oop_static_not_null(src); 424 if (! evac) { 425 return src; 426 } else { 427 return resolve_and_maybe_copy_oop_work(src); 428 } 429 } 430 431 JRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_c2(oopDesc* src)) 432 oop result = ((ShenandoahBarrierSet*) oopDesc::bs())->resolve_and_maybe_copy_oop_work2(src); 433 return (oopDesc*) result; 434 JRT_END 435 436 IRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_interp(oopDesc* src)) 437 oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src)); 438 return (oopDesc*) result; 439 IRT_END 440 441 oop ShenandoahBarrierSet::write_barrier(oop src) { 442 if (ShenandoahWriteBarrier) { 443 if (!oopDesc::is_null(src)) { 444 assert(_heap->is_in(src), "sanity"); 445 assert(src != NULL, "checked before"); 446 oop result = resolve_and_maybe_copy_oopHelper(src); 447 assert(_heap->is_in(result) /*&& result->is_oop()*/, "resolved oop must be NULL, or a valid oop in the heap"); 448 return result; 449 } else { 450 return NULL; 451 } 452 } else { 453 return src; 454 } 455 } 456 457 #ifdef ASSERT 458 void ShenandoahBarrierSet::verify_safe_oop(oop p) { 459 ShenandoahHeap* heap = ShenandoahHeap::heap(); 460 if (p == NULL) return; 461 if (heap->in_collection_set(p) && 462 ! heap->cancelled_concgc()) { 463 tty->print_cr("oop = "PTR_FORMAT", resolved: "PTR_FORMAT", marked-next %s, marked-complete: %s", 464 p2i(p), 465 p2i(read_barrier(p)), 466 BOOL_TO_STR(heap->is_marked_next(p)), 467 BOOL_TO_STR(heap->is_marked_complete(p))); 468 tty->print_cr("in_cset: %s", BOOL_TO_STR(heap->in_collection_set(p))); 469 heap->heap_region_containing((HeapWord*) p)->print(); 470 tty->print_cr("top-at-mark-start: %p", heap->next_top_at_mark_start((HeapWord*) p)); 471 tty->print_cr("top-at-prev-mark-start: %p", heap->complete_top_at_mark_start((HeapWord*) p)); 472 tty->print_cr("marking: %s, evacuating: %s", BOOL_TO_STR(heap->concurrent_mark_in_progress()), BOOL_TO_STR(heap->is_evacuation_in_progress())); 473 assert(false, "We should have fixed this earlier"); 474 } 475 } 476 #endif