1 /* 2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 #include "precompiled.hpp" 24 25 #include "gc/shenandoah/brooksPointer.hpp" 26 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp" 27 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 28 #include "gc/shenandoah/shenandoahHeap.hpp" 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 30 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 31 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" 32 #include "gc/shenandoah/shenandoahRootProcessor.hpp" 33 #include "gc/shenandoah/shenandoahTaskqueue.hpp" 34 #include "gc/shenandoah/shenandoahVerifier.hpp" 35 #include "gc/shenandoah/shenandoahWorkGroup.hpp" 36 #include "memory/allocation.hpp" 37 38 class ShenandoahVerifyOopClosure : public ExtendedOopClosure { 39 private: 40 const char* _phase; 41 ShenandoahVerifier::VerifyOptions _options; 42 ShenandoahVerifierStack* _stack; 43 ShenandoahHeap* _heap; 44 MarkBitMap* _map; 45 ShenandoahLivenessData* _ld; 46 void* _interior_loc; 47 oop _loc; 48 49 public: 50 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld, 51 const char* phase, ShenandoahVerifier::VerifyOptions options) : 52 _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL), 53 _phase(phase), _options(options) {}; 54 55 private: 56 void print_obj(ShenandoahMessageBuffer& msg, oop obj) { 57 ShenandoahHeapRegion *r = _heap->heap_region_containing(obj); 58 stringStream ss; 59 r->print_on(&ss); 60 61 msg.append(" " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name()); 62 msg.append(" %3s allocated after mark start\n", _heap->allocated_after_mark_start((HeapWord *) obj) ? "" : "not"); 63 msg.append(" %3s marked\n", _heap->is_marked(obj) ? "" : "not"); 64 msg.append(" %3s in collection set\n", _heap->in_collection_set(obj) ? "" : "not"); 65 msg.append(" region: %s", ss.as_string()); 66 } 67 68 void print_non_obj(ShenandoahMessageBuffer& msg, void* loc) { 69 msg.append(" outside of Java heap\n"); 70 stringStream ss; 71 os::print_location(&ss, (intptr_t) loc, false); 72 msg.append(" %s\n", ss.as_string()); 73 } 74 75 void print_obj_safe(ShenandoahMessageBuffer& msg, void* loc) { 76 msg.append(" " PTR_FORMAT " - safe print, no details\n", p2i(loc)); 77 if (_heap->is_in(loc)) { 78 ShenandoahHeapRegion* r = _heap->heap_region_containing(loc); 79 if (r != NULL) { 80 stringStream ss; 81 r->print_on(&ss); 82 msg.append(" region: %s", ss.as_string()); 83 } 84 } 85 } 86 87 enum SafeLevel { 88 _safe_unknown, 89 _safe_oop, 90 _safe_oop_fwd, 91 _safe_all, 92 }; 93 94 void print_failure(SafeLevel level, oop obj, const char* label) { 95 ResourceMark rm; 96 97 bool loc_in_heap = (_loc != NULL && _heap->is_in(_loc)); 98 bool interior_loc_in_heap = (_interior_loc != NULL && _heap->is_in(_interior_loc)); 99 100 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label); 101 102 msg.append("Referenced from:\n"); 103 if (_interior_loc != NULL) { 104 msg.append(" interior location: " PTR_FORMAT "\n", p2i(_interior_loc)); 105 if (loc_in_heap) { 106 print_obj(msg, _loc); 107 } else { 108 print_non_obj(msg, _interior_loc); 109 } 110 } else { 111 msg.append(" no location recorded, probably a plain heap scan\n"); 112 } 113 msg.append("\n"); 114 115 msg.append("Object:\n"); 116 if (level >= _safe_oop) { 117 print_obj(msg, obj); 118 } else { 119 print_obj_safe(msg, obj); 120 } 121 msg.append("\n"); 122 123 if (level >= _safe_oop) { 124 oop fwd = (oop) BrooksPointer::get_raw(obj); 125 if (!oopDesc::unsafe_equals(obj, fwd)) { 126 msg.append("Forwardee:\n"); 127 if (level >= _safe_oop_fwd) { 128 print_obj(msg, fwd); 129 } else { 130 print_obj_safe(msg, fwd); 131 } 132 msg.append("\n"); 133 } 134 } 135 136 if (level >= _safe_oop_fwd) { 137 oop fwd = (oop) BrooksPointer::get_raw(obj); 138 oop fwd2 = (oop) BrooksPointer::get_raw(fwd); 139 if (!oopDesc::unsafe_equals(fwd, fwd2)) { 140 msg.append("Second forwardee:\n"); 141 print_obj_safe(msg, fwd2); 142 msg.append("\n"); 143 } 144 } 145 146 if (loc_in_heap && UseShenandoahMatrix && (level == _safe_all)) { 147 msg.append("Matrix connections:\n"); 148 149 oop fwd_to = (oop) BrooksPointer::get_raw(obj); 150 oop fwd_from = (oop) BrooksPointer::get_raw(_loc); 151 152 size_t from_idx = _heap->heap_region_index_containing(_loc); 153 size_t to_idx = _heap->heap_region_index_containing(obj); 154 size_t fwd_from_idx = _heap->heap_region_index_containing(fwd_from); 155 size_t fwd_to_idx = _heap->heap_region_index_containing(fwd_to); 156 157 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix(); 158 msg.append(" %35s %3s connected\n", 159 "reference and object", 160 matrix->is_connected(from_idx, to_idx) ? "" : "not"); 161 msg.append(" %35s %3s connected\n", 162 "fwd(reference) and object", 163 matrix->is_connected(fwd_from_idx, to_idx) ? "" : "not"); 164 msg.append(" %35s %3s connected\n", 165 "reference and fwd(object)", 166 matrix->is_connected(from_idx, fwd_to_idx) ? "" : "not"); 167 msg.append(" %35s %3s connected\n", 168 "fwd(reference) and fwd(object)", 169 matrix->is_connected(fwd_from_idx, fwd_to_idx) ? "" : "not"); 170 171 if (interior_loc_in_heap) { 172 size_t from_interior_idx = _heap->heap_region_index_containing(_interior_loc); 173 msg.append(" %35s %3s connected\n", 174 "interior-reference and object", 175 matrix->is_connected(from_interior_idx, to_idx) ? "" : "not"); 176 msg.append(" %35s %3s connected\n", 177 "interior-reference and fwd(object)", 178 matrix->is_connected(from_interior_idx, fwd_to_idx) ? "" : "not"); 179 } 180 } 181 182 report_vm_error(__FILE__, __LINE__, msg.buffer()); 183 } 184 185 void verify(SafeLevel level, oop obj, bool test, const char* label) { 186 if (!test) { 187 print_failure(level, obj, label); 188 } 189 } 190 191 template <class T> 192 void do_oop_work(T* p) { 193 T o = oopDesc::load_heap_oop(p); 194 if (!oopDesc::is_null(o)) { 195 oop obj = oopDesc::decode_heap_oop_not_null(o); 196 197 // Single threaded verification can use faster non-atomic stack and bitmap 198 // methods. 199 // 200 // For performance reasons, only fully verify non-marked field values. 201 // We are here when the host object for *p is already marked. If field value 202 // is marked already, then we have to verify the matrix connection between 203 // host object and field value, regardless. 204 205 HeapWord* addr = (HeapWord*) obj; 206 if (_map->parMark(addr)) { 207 verify_oop_at(p, obj); 208 _stack->push(ShenandoahVerifierTask(obj)); 209 } else { 210 verify_matrix(p, obj); 211 } 212 } 213 } 214 215 void verify_oop(oop obj) { 216 // Perform consistency checks with gradually decreasing safety level. This guarantees 217 // that failure report would not try to touch something that was not yet verified to be 218 // safe to process. 219 220 verify(_safe_unknown, obj, _heap->is_in(obj), 221 "oop must be in heap"); 222 verify(_safe_unknown, obj, check_obj_alignment(obj), 223 "oop must be aligned"); 224 225 // Verify that obj is not in dead space: 226 { 227 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); 228 229 HeapWord *obj_addr = (HeapWord *) obj; 230 verify(_safe_unknown, obj, obj_addr < obj_reg->top(), 231 "Object start should be within the region"); 232 233 if (!obj_reg->is_humongous()) { 234 verify(_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), 235 "Object end should be within the region"); 236 } else { 237 size_t humongous_start = obj_reg->region_number(); 238 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift()); 239 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) { 240 verify(_safe_unknown, obj, _heap->regions()->get(idx)->is_humongous_continuation(), 241 "Humongous object is in continuation that fits it"); 242 } 243 } 244 245 verify(_safe_unknown, obj, Metaspace::contains(obj->klass()), 246 "klass pointer must go to metaspace"); 247 248 // ------------ obj is safe at this point -------------- 249 250 verify(_safe_oop, obj, obj_reg->is_active(), 251 "Object should be in active region"); 252 253 switch (_options._verify_liveness) { 254 case ShenandoahVerifier::_verify_liveness_disable: 255 // skip 256 break; 257 case ShenandoahVerifier::_verify_liveness_complete: 258 Atomic::add(obj->size() + BrooksPointer::word_size(), &_ld[obj_reg->region_number()]); 259 // fallthrough for fast failure for un-live regions: 260 case ShenandoahVerifier::_verify_liveness_conservative: 261 verify(_safe_oop, obj, obj_reg->has_live(), 262 "Object must belong to region with live data"); 263 break; 264 default: 265 assert(false, "Unhandled liveness verification"); 266 } 267 } 268 269 oop fwd = (oop) BrooksPointer::get_raw(obj); 270 if (!oopDesc::unsafe_equals(obj, fwd)) { 271 verify(_safe_oop, obj, _heap->is_in(fwd), 272 "Forwardee must be in heap"); 273 verify(_safe_oop, obj, !oopDesc::is_null(fwd), 274 "Forwardee is set"); 275 verify(_safe_oop, obj, check_obj_alignment(fwd), 276 "Forwardee must be aligned"); 277 278 // Verify that forwardee is not in the dead space: 279 if (!oopDesc::unsafe_equals(obj, fwd)) { 280 ShenandoahHeapRegion *fwd_reg = _heap->heap_region_containing(fwd); 281 verify(_safe_oop, obj, !fwd_reg->is_humongous(), 282 "Should have no humongous forwardees"); 283 284 HeapWord *fwd_addr = (HeapWord *) fwd; 285 verify(_safe_oop, obj, fwd_addr < fwd_reg->top(), 286 "Forwardee start should be within the region"); 287 verify(_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), 288 "Forwardee end should be within the region"); 289 } 290 291 verify(_safe_oop, obj, Metaspace::contains(fwd->klass()), 292 "Forwardee klass pointer must go to metaspace"); 293 verify(_safe_oop, obj, obj->klass() == fwd->klass(), 294 "Forwardee klass pointer must go to metaspace"); 295 296 oop fwd2 = (oop) BrooksPointer::get_raw(fwd); 297 verify(_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2), 298 "Double forwarding"); 299 } 300 301 // ------------ obj and fwd are safe at this point -------------- 302 303 switch (_options._verify_marked) { 304 case ShenandoahVerifier::_verify_marked_disable: 305 // skip 306 break; 307 case ShenandoahVerifier::_verify_marked: 308 verify(_safe_all, obj, _heap->is_marked(obj), 309 "Must be marked"); 310 break; 311 default: 312 assert(false, "Unhandled mark verification"); 313 } 314 315 switch (_options._verify_forwarded) { 316 case ShenandoahVerifier::_verify_forwarded_disable: 317 // skip 318 break; 319 case ShenandoahVerifier::_verify_forwarded_none: { 320 verify(_safe_all, obj, oopDesc::unsafe_equals(obj, fwd), 321 "Should not be forwarded"); 322 break; 323 } 324 case ShenandoahVerifier::_verify_forwarded_allow: { 325 if (!oopDesc::unsafe_equals(obj, fwd)) { 326 verify(_safe_all, obj, _heap->heap_region_containing(obj) != _heap->heap_region_containing(fwd), 327 "Forwardee should be in another region"); 328 } 329 break; 330 } 331 default: 332 assert(false, "Unhandled forwarding verification"); 333 } 334 335 switch (_options._verify_cset) { 336 case ShenandoahVerifier::_verify_cset_disable: 337 // skip 338 break; 339 case ShenandoahVerifier::_verify_cset_none: 340 verify(_safe_all, obj, !_heap->in_collection_set(obj), 341 "Should not have references to collection set"); 342 break; 343 case ShenandoahVerifier::_verify_cset_forwarded: 344 if (_heap->in_collection_set(obj)) { 345 verify(_safe_all, obj, !oopDesc::unsafe_equals(obj, fwd), 346 "Object in collection set, should have forwardee"); 347 } 348 break; 349 default: 350 assert(false, "Unhandled cset verification"); 351 } 352 353 verify_matrix(_interior_loc, obj); 354 } 355 356 void verify_matrix(void* interior, oop obj) { 357 if (!UseShenandoahMatrix || !_heap->is_in(interior)) return; 358 switch (_options._verify_matrix) { 359 case ShenandoahVerifier::_verify_matrix_conservative: { 360 size_t from_idx = _heap->heap_region_index_containing(interior); 361 size_t to_idx = _heap->heap_region_index_containing(obj); 362 _interior_loc = interior; 363 verify(_safe_all, obj, _heap->connection_matrix()->is_connected(from_idx, to_idx), 364 "Must be connected"); 365 _interior_loc = NULL; 366 break; 367 } 368 case ShenandoahVerifier::_verify_matrix_disable: 369 break; 370 default: 371 assert(false, "Unhandled matrix verification"); 372 } 373 } 374 375 public: 376 377 /** 378 * Verify object with known interior reference. 379 * @param p interior reference where the object is referenced from; can be off-heap 380 * @param obj verified object 381 */ 382 template <class T> 383 void verify_oop_at(T* p, oop obj) { 384 _interior_loc = p; 385 verify_oop(obj); 386 _interior_loc = NULL; 387 } 388 389 /** 390 * Verify object without known interior reference. 391 * Useful when picking up the object at known offset in heap, 392 * but without knowing what objects reference it. 393 * @param obj verified object 394 */ 395 void verify_oop_standalone(oop obj) { 396 _interior_loc = NULL; 397 verify_oop(obj); 398 _interior_loc = NULL; 399 } 400 401 /** 402 * Verify oop fields from this object. 403 * @param obj host object for verified fields 404 */ 405 void verify_oops_from(oop obj) { 406 _loc = obj; 407 obj->oop_iterate(this); 408 _loc = NULL; 409 } 410 411 void do_oop(oop* p) { do_oop_work(p); } 412 void do_oop(narrowOop* p) { do_oop_work(p); } 413 }; 414 415 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { 416 private: 417 size_t _used, _committed, _garbage; 418 public: 419 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {}; 420 421 bool heap_region_do(ShenandoahHeapRegion* r) { 422 _used += r->used(); 423 _garbage += r->garbage(); 424 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; 425 return false; 426 } 427 428 size_t used() { return _used; } 429 size_t committed() { return _committed; } 430 size_t garbage() { return _garbage; } 431 }; 432 433 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { 434 private: 435 ShenandoahHeap* _heap; 436 const char* _phase; 437 ShenandoahVerifier::VerifyRegions _regions; 438 public: 439 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) : 440 _heap(ShenandoahHeap::heap()), _regions(regions), _phase(phase) {}; 441 442 void print_failure(ShenandoahHeapRegion* r, const char* label) { 443 ResourceMark rm; 444 445 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label); 446 447 stringStream ss; 448 r->print_on(&ss); 449 msg.append("%s", ss.as_string()); 450 451 report_vm_error(__FILE__, __LINE__, msg.buffer()); 452 } 453 454 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) { 455 if (!test) { 456 print_failure(r, msg); 457 } 458 } 459 460 bool heap_region_do(ShenandoahHeapRegion* r) { 461 switch (_regions) { 462 case ShenandoahVerifier::_verify_regions_disable: 463 break; 464 case ShenandoahVerifier::_verify_regions_notrash: 465 verify(r, !r->is_trash(), 466 "Should not have trash regions"); 467 break; 468 case ShenandoahVerifier::_verify_regions_nocset: 469 verify(r, !r->is_cset(), 470 "Should not have cset regions"); 471 break; 472 case ShenandoahVerifier::_verify_regions_notrash_nocset: 473 verify(r, !r->is_trash(), 474 "Should not have trash regions"); 475 verify(r, !r->is_cset(), 476 "Should not have cset regions"); 477 break; 478 default: 479 ShouldNotReachHere(); 480 } 481 482 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(), 483 "Capacity should match region size"); 484 485 verify(r, r->bottom() <= _heap->top_at_mark_start(r->bottom()), 486 "Region top should not be less than bottom"); 487 488 verify(r, _heap->top_at_mark_start(r->bottom()) <= r->top(), 489 "TAMS should not be larger than top"); 490 491 verify(r, r->get_live_data_bytes() <= r->capacity(), 492 "Live data cannot be larger than capacity"); 493 494 verify(r, r->garbage() <= r->capacity(), 495 "Garbage cannot be larger than capacity"); 496 497 verify(r, r->used() <= r->capacity(), 498 "Used cannot be larger than capacity"); 499 500 verify(r, r->get_shared_allocs() <= r->capacity(), 501 "Shared alloc count should not be larger than capacity"); 502 503 verify(r, r->get_tlab_allocs() <= r->capacity(), 504 "TLAB alloc count should not be larger than capacity"); 505 506 verify(r, r->get_gclab_allocs() <= r->capacity(), 507 "GCLAB alloc count should not be larger than capacity"); 508 509 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(), 510 "Accurate accounting: shared + TLAB + GCLAB = used"); 511 512 verify(r, !r->is_empty() || !r->has_live(), 513 "Empty regions should not have live data"); 514 515 verify(r, r->is_cset() == r->in_collection_set(), 516 "Transitional: region flags and collection set agree"); 517 518 verify(r, r->is_empty() || r->first_alloc_seq_num() != 0, 519 "Non-empty regions should have first timestamp set"); 520 521 verify(r, r->is_empty() || r->last_alloc_seq_num() != 0, 522 "Non-empty regions should have last timestamp set"); 523 524 verify(r, r->first_alloc_seq_num() <= r->last_alloc_seq_num(), 525 "First timestamp should not be greater than last timestamp"); 526 527 return false; 528 } 529 }; 530 531 class ShenandoahVerifierReachableTask : public AbstractGangTask { 532 private: 533 const char* _label; 534 ShenandoahRootProcessor* _rp; 535 ShenandoahVerifier::VerifyOptions _options; 536 ShenandoahHeap* _heap; 537 ShenandoahLivenessData* _ld; 538 MarkBitMap* _bitmap; 539 volatile size_t _processed; 540 541 public: 542 ShenandoahVerifierReachableTask(MarkBitMap* bitmap, 543 ShenandoahLivenessData* ld, 544 ShenandoahRootProcessor* rp, 545 const char* label, 546 ShenandoahVerifier::VerifyOptions options) : 547 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"), 548 _heap(ShenandoahHeap::heap()), _rp(rp), _ld(ld), _bitmap(bitmap), _processed(0), 549 _label(label), _options(options) {}; 550 551 size_t processed() { 552 return _processed; 553 } 554 555 virtual void work(uint worker_id) { 556 ResourceMark rm; 557 ShenandoahVerifierStack stack; 558 559 // On level 2, we need to only check the roots once. 560 // On level 3, we want to check the roots, and seed the local stack. 561 // It is a lesser evil to accept multiple root scans at level 3, because 562 // extended parallelism would buy us out. 563 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0)) 564 || (ShenandoahVerifyLevel >= 3)) { 565 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 566 ShenandoahMessageBuffer("%s, Roots", _label), 567 _options); 568 _rp->process_all_roots_slow(&cl); 569 } 570 571 size_t processed = 0; 572 573 if (ShenandoahVerifyLevel >= 3) { 574 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 575 ShenandoahMessageBuffer("%s, Reachable", _label), 576 _options); 577 while (!stack.is_empty()) { 578 processed++; 579 ShenandoahVerifierTask task = stack.pop(); 580 cl.verify_oops_from(task.obj()); 581 } 582 } 583 584 Atomic::add(processed, &_processed); 585 } 586 }; 587 588 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { 589 private: 590 const char* _label; 591 ShenandoahVerifier::VerifyOptions _options; 592 ShenandoahHeap *_heap; 593 ShenandoahHeapRegionSet* _regions; 594 MarkBitMap* _bitmap; 595 ShenandoahLivenessData* _ld; 596 volatile size_t _claimed; 597 volatile size_t _processed; 598 599 public: 600 ShenandoahVerifierMarkedRegionTask(ShenandoahHeapRegionSet* regions, MarkBitMap* bitmap, 601 ShenandoahLivenessData* ld, 602 const char* label, 603 ShenandoahVerifier::VerifyOptions options) : 604 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"), 605 _heap(ShenandoahHeap::heap()), _regions(regions), _ld(ld), _bitmap(bitmap), _claimed(0), _processed(0), 606 _label(label), _options(options) {}; 607 608 size_t processed() { 609 return _processed; 610 } 611 612 virtual void work(uint worker_id) { 613 ShenandoahVerifierStack stack; 614 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 615 ShenandoahMessageBuffer("%s, Marked", _label), 616 _options); 617 618 while (true) { 619 size_t v = Atomic::add(1u, &_claimed) - 1; 620 if (v < _heap->num_regions()) { 621 ShenandoahHeapRegion* r = _regions->get(v); 622 if (!r->is_humongous() && !r->is_trash()) { 623 work_regular(r, stack, cl); 624 } else if (r->is_humongous_start()) { 625 work_humongous(r, stack, cl); 626 } 627 } else { 628 break; 629 } 630 } 631 } 632 633 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { 634 size_t processed = 0; 635 HeapWord* obj = r->bottom() + BrooksPointer::word_size(); 636 if (_heap->is_marked((oop)obj)) { 637 verify_and_follow(obj, stack, cl, &processed); 638 } 639 Atomic::add(processed, &_processed); 640 } 641 642 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { 643 size_t processed = 0; 644 MarkBitMap* mark_bit_map = _heap->mark_bit_map(); 645 HeapWord* tams = _heap->top_at_mark_start(r->bottom()); 646 647 // Bitmaps, before TAMS 648 if (tams > r->bottom()) { 649 HeapWord* start = r->bottom() + BrooksPointer::word_size(); 650 HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams); 651 652 while (addr < tams) { 653 verify_and_follow(addr, stack, cl, &processed); 654 addr += BrooksPointer::word_size(); 655 if (addr < tams) { 656 addr = mark_bit_map->getNextMarkedWordAddress(addr, tams); 657 } 658 } 659 } 660 661 // Size-based, after TAMS 662 { 663 HeapWord* limit = r->top(); 664 HeapWord* addr = tams + BrooksPointer::word_size(); 665 666 while (addr < limit) { 667 verify_and_follow(addr, stack, cl, &processed); 668 addr += oop(addr)->size() + BrooksPointer::word_size(); 669 } 670 } 671 672 Atomic::add(processed, &_processed); 673 } 674 675 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) { 676 if (!_bitmap->parMark(addr)) return; 677 678 // Verify the object itself: 679 oop obj = oop(addr); 680 cl.verify_oop_standalone(obj); 681 682 // Verify everything reachable from that object too, hopefully realizing 683 // everything was already marked, and never touching further: 684 cl.verify_oops_from(obj); 685 (*processed)++; 686 687 while (!stack.is_empty()) { 688 ShenandoahVerifierTask task = stack.pop(); 689 cl.verify_oops_from(task.obj()); 690 (*processed)++; 691 } 692 } 693 }; 694 695 void ShenandoahVerifier::verify_at_safepoint(const char *label, 696 VerifyForwarded forwarded, VerifyMarked marked, 697 VerifyMatrix matrix, VerifyCollectionSet cset, 698 VerifyLiveness liveness, VerifyRegions regions) { 699 guarantee(SafepointSynchronize::is_at_safepoint(), "only when nothing else happens"); 700 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize"); 701 702 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check 703 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/); 704 705 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel); 706 707 // Heap size checks 708 { 709 ShenandoahHeapLocker lock(_heap->lock()); 710 711 ShenandoahCalculateRegionStatsClosure cl; 712 _heap->heap_region_iterate(&cl); 713 size_t heap_used = _heap->used(); 714 guarantee(cl.used() == heap_used, 715 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "K, regions-used = " SIZE_FORMAT "K", 716 label, heap_used/K, cl.used()/K); 717 718 size_t heap_committed = _heap->committed(); 719 guarantee(cl.committed() == heap_committed, 720 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "K, regions-committed = " SIZE_FORMAT "K", 721 label, heap_committed/K, cl.committed()/K); 722 } 723 724 // Internal heap region checks 725 if (ShenandoahVerifyLevel >= 1) { 726 ShenandoahVerifyHeapRegionClosure cl(label, regions); 727 _heap->heap_region_iterate(&cl, 728 /* skip_cset = */ false, 729 /* skip_humongous_cont = */ false); 730 } 731 732 OrderAccess::fence(); 733 _heap->make_tlabs_parsable(false); 734 735 // Allocate temporary bitmap for storing marking wavefront: 736 MemRegion mr = MemRegion(_verification_bit_map->startWord(), _verification_bit_map->endWord()); 737 _verification_bit_map->clear_range_large(mr); 738 739 // Allocate temporary array for storing liveness data 740 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC); 741 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0); 742 743 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, matrix, cset, liveness, regions); 744 745 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap. 746 // This verifies what application can see, since it only cares about reachable objects. 747 size_t count_reachable = 0; 748 if (ShenandoahVerifyLevel >= 2) { 749 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), 750 ShenandoahPhaseTimings::_num_phases); // no need for stats 751 752 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options); 753 _heap->workers()->run_task(&task); 754 count_reachable = task.processed(); 755 } 756 757 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector, 758 // not the application, can see during the region scans. There is no reason to process the objects 759 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS: 760 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics 761 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect) 762 // version 763 764 size_t count_marked = 0; 765 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked) { 766 ShenandoahVerifierMarkedRegionTask task(_heap->regions(), _verification_bit_map, ld, label, options); 767 _heap->workers()->run_task(&task); 768 count_marked = task.processed(); 769 } else { 770 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_disable, "Should be"); 771 } 772 773 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes 774 // marked objects. 775 776 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked && liveness == _verify_liveness_complete) { 777 ShenandoahHeapRegionSet* set = _heap->regions(); 778 for (size_t i = 0; i < _heap->num_regions(); i++) { 779 ShenandoahHeapRegion* r = set->get(i); 780 781 juint verf_live = 0; 782 if (r->is_humongous()) { 783 // For humongous objects, test if start region is marked live, and if so, 784 // all humongous regions in that chain have live data equal to their "used". 785 juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]); 786 if (start_live > 0) { 787 verf_live = (juint)(r->used() / HeapWordSize); 788 } 789 } else { 790 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]); 791 } 792 793 size_t reg_live = r->get_live_data_words(); 794 if (reg_live != verf_live) { 795 ResourceMark rm; 796 stringStream ss; 797 r->print_on(&ss); 798 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s", 799 label, reg_live, verf_live, ss.as_string()); 800 } 801 } 802 } 803 804 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)", 805 label, ShenandoahVerifyLevel, count_reachable, count_marked); 806 807 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld); 808 } 809 810 void ShenandoahVerifier::verify_generic(VerifyOption vo) { 811 verify_at_safepoint( 812 "Generic Verification", 813 _verify_forwarded_allow, // conservatively allow forwarded 814 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 815 _verify_matrix_disable, // matrix can be inconsistent here 816 _verify_cset_disable, // cset may be inconsistent 817 _verify_liveness_disable, // no reliable liveness data 818 _verify_regions_disable // no reliable region data 819 ); 820 } 821 822 void ShenandoahVerifier::verify_before_concmark() { 823 if (_heap->need_update_refs()) { 824 verify_at_safepoint( 825 "Before Mark", 826 _verify_forwarded_allow, // may have forwarded references 827 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 828 _verify_matrix_disable, // matrix is foobared 829 _verify_cset_forwarded, // allow forwarded references to cset 830 _verify_liveness_disable, // no reliable liveness data 831 _verify_regions_notrash // no trash regions 832 ); 833 } else { 834 verify_at_safepoint( 835 "Before Mark", 836 _verify_forwarded_none, // UR should have fixed up 837 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 838 _verify_matrix_conservative, // UR should have fixed matrix 839 _verify_cset_none, // UR should have fixed this 840 _verify_liveness_disable, // no reliable liveness data 841 _verify_regions_notrash // no trash regions 842 ); 843 } 844 } 845 846 void ShenandoahVerifier::verify_after_concmark() { 847 verify_at_safepoint( 848 "After Mark", 849 _verify_forwarded_none, // no forwarded references 850 _verify_marked, // bitmaps as precise as we can get 851 _verify_matrix_disable, // matrix might be foobared 852 _verify_cset_none, // no references to cset anymore 853 _verify_liveness_complete, // liveness data must be complete here 854 _verify_regions_disable // trash regions not yet recycled 855 ); 856 } 857 858 void ShenandoahVerifier::verify_before_evacuation() { 859 // Evacuation is always preceded by mark, but we want to have a sanity check after 860 // selecting the collection set, and (immediate) regions recycling 861 verify_at_safepoint( 862 "Before Evacuation", 863 _verify_forwarded_none, // no forwarded references 864 _verify_marked, // walk over marked objects too 865 _verify_matrix_disable, // skip, verified after mark 866 _verify_cset_disable, // skip, verified after mark 867 _verify_liveness_disable, // skip, verified after mark 868 _verify_regions_disable // trash regions not yet recycled 869 ); 870 } 871 872 void ShenandoahVerifier::verify_after_evacuation() { 873 verify_at_safepoint( 874 "After Evacuation", 875 _verify_forwarded_allow, // objects are still forwarded 876 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well 877 _verify_matrix_disable, // matrix is inconsistent here 878 _verify_cset_forwarded, // all cset refs are fully forwarded 879 _verify_liveness_disable, // no reliable liveness data anymore 880 _verify_regions_notrash // trash regions have been recycled already 881 ); 882 } 883 884 void ShenandoahVerifier::verify_before_updaterefs() { 885 verify_at_safepoint( 886 "Before Updating References", 887 _verify_forwarded_allow, // forwarded references allowed 888 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well 889 _verify_matrix_disable, // matrix is inconsistent here 890 _verify_cset_forwarded, // all cset refs are fully forwarded 891 _verify_liveness_disable, // no reliable liveness data anymore 892 _verify_regions_notrash // trash regions have been recycled already 893 ); 894 } 895 896 void ShenandoahVerifier::verify_after_updaterefs() { 897 verify_at_safepoint( 898 "After Updating References", 899 _verify_forwarded_none, // no forwarded references 900 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well 901 _verify_matrix_conservative, // matrix is conservatively consistent 902 _verify_cset_none, // no cset references, all updated 903 _verify_liveness_disable, // no reliable liveness data anymore 904 _verify_regions_nocset // no cset regions, trash regions have appeared 905 ); 906 } 907 908 void ShenandoahVerifier::verify_before_partial() { 909 verify_at_safepoint( 910 "Before Partial GC", 911 _verify_forwarded_none, // cannot have forwarded objects 912 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well 913 _verify_matrix_conservative, // matrix is conservatively consistent 914 _verify_cset_none, // no cset references before partial 915 _verify_liveness_disable, // no reliable liveness data anymore 916 _verify_regions_notrash_nocset // no trash and no cset regions 917 ); 918 } 919 920 void ShenandoahVerifier::verify_after_partial() { 921 verify_at_safepoint( 922 "After Partial GC", 923 _verify_forwarded_none, // cannot have forwarded objects 924 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well 925 _verify_matrix_conservative, // matrix is conservatively consistent 926 _verify_cset_none, // no cset references left after partial 927 _verify_liveness_disable, // no reliable liveness data anymore 928 _verify_regions_nocset // no cset regions, trash regions allowed 929 ); 930 } 931 932 void ShenandoahVerifier::verify_before_fullgc() { 933 verify_at_safepoint( 934 "Before Full GC", 935 _verify_forwarded_allow, // can have forwarded objects 936 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 937 _verify_matrix_disable, // matrix might be foobared 938 _verify_cset_disable, // cset might be foobared 939 _verify_liveness_disable, // no reliable liveness data anymore 940 _verify_regions_disable // no reliable region data here 941 ); 942 } 943 944 void ShenandoahVerifier::verify_after_fullgc() { 945 verify_at_safepoint( 946 "After Full GC", 947 _verify_forwarded_none, // all objects are non-forwarded 948 _verify_marked, // all objects are marked 949 _verify_matrix_conservative, // matrix is conservatively consistent 950 _verify_cset_none, // no cset references 951 _verify_liveness_disable, // no reliable liveness data anymore 952 _verify_regions_notrash_nocset // no trash, no cset 953 ); 954 } 955 956 void ShenandoahVerifier::verify_oop_fwdptr(oop obj, oop fwd) { 957 guarantee(UseShenandoahGC, "must only be called when Shenandoah is used"); 958 959 ShenandoahHeap* heap = ShenandoahHeap::heap(); 960 961 guarantee(obj != NULL, "oop is not NULL"); 962 guarantee(heap->is_in(obj), "oop must point to a heap address: " PTR_FORMAT, p2i(obj)); 963 964 guarantee(fwd != NULL, "forwardee is not NULL"); 965 if (!heap->is_in(fwd)) { 966 ResourceMark rm; 967 ShenandoahHeapRegion* r = heap->heap_region_containing(obj); 968 stringStream obj_region; 969 r->print_on(&obj_region); 970 971 fatal("forwardee must point to a heap address: " PTR_FORMAT " -> " PTR_FORMAT "\n %s", 972 p2i(obj), p2i(fwd), obj_region.as_string()); 973 } 974 975 if (!oopDesc::unsafe_equals(fwd, obj) && 976 (heap->heap_region_containing(fwd) == 977 heap->heap_region_containing(obj))) { 978 ResourceMark rm; 979 ShenandoahHeapRegion* ro = heap->heap_region_containing(obj); 980 stringStream obj_region; 981 ro->print_on(&obj_region); 982 983 ShenandoahHeapRegion* rf = heap->heap_region_containing(fwd); 984 stringStream fwd_region; 985 rf->print_on(&fwd_region); 986 987 fatal("forwardee should be self, or another region: " PTR_FORMAT " -> " PTR_FORMAT "\n %s %s", 988 p2i(obj), p2i(fwd), 989 obj_region.as_string(), fwd_region.as_string()); 990 } 991 992 if (!oopDesc::unsafe_equals(obj, fwd)) { 993 oop fwd2 = oop(BrooksPointer::get_raw(fwd)); 994 if (!oopDesc::unsafe_equals(fwd, fwd2)) { 995 // We should never be forwarded more than once. 996 ResourceMark rm; 997 998 ShenandoahHeapRegion* ro = heap->heap_region_containing(obj); 999 stringStream obj_region; 1000 ro->print_on(&obj_region); 1001 1002 ShenandoahHeapRegion* rf = heap->heap_region_containing(fwd); 1003 stringStream fwd_region; 1004 rf->print_on(&fwd_region); 1005 1006 ShenandoahHeapRegion* rf2 = heap->heap_region_containing(fwd2); 1007 stringStream fwd2_region; 1008 rf2->print_on(&fwd2_region); 1009 1010 fatal("Multiple forwardings: " PTR_FORMAT " -> " PTR_FORMAT " -> " PTR_FORMAT "\n %s %s %s", 1011 p2i(obj), p2i(fwd), p2i(fwd2), 1012 obj_region.as_string(), fwd_region.as_string(), fwd2_region.as_string()); 1013 } 1014 } 1015 } 1016 1017 void ShenandoahVerifier::verify_oop(oop obj) { 1018 oop fwd = oop(BrooksPointer::get_raw(obj)); 1019 verify_oop_fwdptr(obj, fwd); 1020 }