1 /* 2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 26 #include "memory/allocation.hpp" 27 #include "gc_implementation/shenandoah/shenandoahAsserts.hpp" 28 #include "gc_implementation/shenandoah/shenandoahHeap.hpp" 29 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" 30 #include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp" 31 #include "gc_implementation/shenandoah/shenandoahVerifier.hpp" 32 #include "gc_implementation/shenandoah/brooksPointer.hpp" 33 #include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp" 34 35 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp) 36 #ifdef verify_oop 37 #undef verify_oop 38 #endif 39 40 class ShenandoahVerifyOopClosure : public ExtendedOopClosure { 41 private: 42 const char* _phase; 43 ShenandoahVerifier::VerifyOptions _options; 44 ShenandoahVerifierStack* _stack; 45 ShenandoahHeap* _heap; 46 MarkBitMap* _map; 47 ShenandoahLivenessData* _ld; 48 void* _interior_loc; 49 oop _loc; 50 51 public: 52 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld, 53 const char* phase, ShenandoahVerifier::VerifyOptions options) : 54 _phase(phase), 55 _options(options), 56 _stack(stack), 57 _heap(ShenandoahHeap::heap()), 58 _map(map), 59 _ld(ld), 60 _interior_loc(NULL), 61 _loc(NULL) { } 62 63 private: 64 void verify(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) { 65 if (!test) { 66 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__); 67 } 68 } 69 70 template <class T> 71 void do_oop_work(T* p) { 72 T o = oopDesc::load_heap_oop(p); 73 if (!oopDesc::is_null(o)) { 74 oop obj = oopDesc::decode_heap_oop_not_null(o); 75 76 // Single threaded verification can use faster non-atomic stack and bitmap 77 // methods. 78 // 79 // For performance reasons, only fully verify non-marked field values. 80 // We are here when the host object for *p is already marked. 81 82 HeapWord* addr = (HeapWord*) obj; 83 if (_map->parMark(addr)) { 84 verify_oop_at(p, obj); 85 _stack->push(ShenandoahVerifierTask(obj)); 86 } 87 } 88 } 89 90 void verify_oop(oop obj) { 91 // Perform consistency checks with gradually decreasing safety level. This guarantees 92 // that failure report would not try to touch something that was not yet verified to be 93 // safe to process. 94 95 verify(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj), 96 "oop must be in heap"); 97 verify(ShenandoahAsserts::_safe_unknown, obj, check_obj_alignment(obj), 98 "oop must be aligned"); 99 100 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); 101 Klass* obj_klass = obj->klass_or_null(); 102 103 // Verify that obj is not in dead space: 104 { 105 // Do this before touching obj->size() 106 verify(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL, 107 "Object klass pointer should not be NULL"); 108 verify(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass), 109 "Object klass pointer must go to metaspace"); 110 111 HeapWord *obj_addr = (HeapWord *) obj; 112 verify(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(), 113 "Object start should be within the region"); 114 115 if (!obj_reg->is_humongous()) { 116 verify(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), 117 "Object end should be within the region"); 118 } else { 119 size_t humongous_start = obj_reg->region_number(); 120 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift()); 121 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) { 122 verify(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(), 123 "Humongous object is in continuation that fits it"); 124 } 125 } 126 127 // ------------ obj is safe at this point -------------- 128 129 verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(), 130 "Object should be in active region"); 131 132 switch (_options._verify_liveness) { 133 case ShenandoahVerifier::_verify_liveness_disable: 134 // skip 135 break; 136 case ShenandoahVerifier::_verify_liveness_complete: 137 Atomic::add(obj->size() + BrooksPointer::word_size(), &_ld[obj_reg->region_number()]); 138 // fallthrough for fast failure for un-live regions: 139 case ShenandoahVerifier::_verify_liveness_conservative: 140 verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(), 141 "Object must belong to region with live data"); 142 break; 143 default: 144 assert(false, "Unhandled liveness verification"); 145 } 146 } 147 148 oop fwd = (oop) BrooksPointer::get_raw_unchecked(obj); 149 150 ShenandoahHeapRegion* fwd_reg = NULL; 151 152 if (!oopDesc::unsafe_equals(obj, fwd)) { 153 verify(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd), 154 "Forwardee must be in heap"); 155 verify(ShenandoahAsserts::_safe_oop, obj, !oopDesc::is_null(fwd), 156 "Forwardee is set"); 157 verify(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd), 158 "Forwardee must be aligned"); 159 160 // Do this before touching fwd->size() 161 Klass* fwd_klass = fwd->klass_or_null(); 162 verify(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL, 163 "Forwardee klass pointer should not be NULL"); 164 verify(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass), 165 "Forwardee klass pointer must go to metaspace"); 166 verify(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass, 167 "Forwardee klass pointer must go to metaspace"); 168 169 fwd_reg = _heap->heap_region_containing(fwd); 170 171 // Verify that forwardee is not in the dead space: 172 verify(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(), 173 "Should have no humongous forwardees"); 174 175 HeapWord *fwd_addr = (HeapWord *) fwd; 176 verify(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), 177 "Forwardee start should be within the region"); 178 verify(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), 179 "Forwardee end should be within the region"); 180 181 oop fwd2 = (oop) BrooksPointer::get_raw_unchecked(fwd); 182 verify(ShenandoahAsserts::_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2), 183 "Double forwarding"); 184 } else { 185 fwd_reg = obj_reg; 186 } 187 188 // ------------ obj and fwd are safe at this point -------------- 189 190 switch (_options._verify_marked) { 191 case ShenandoahVerifier::_verify_marked_disable: 192 // skip 193 break; 194 case ShenandoahVerifier::_verify_marked_next: 195 verify(ShenandoahAsserts::_safe_all, obj, _heap->next_marking_context()->is_marked(obj), 196 "Must be marked in next bitmap"); 197 break; 198 case ShenandoahVerifier::_verify_marked_complete: 199 verify(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj), 200 "Must be marked in complete bitmap"); 201 break; 202 default: 203 assert(false, "Unhandled mark verification"); 204 } 205 206 switch (_options._verify_forwarded) { 207 case ShenandoahVerifier::_verify_forwarded_disable: 208 // skip 209 break; 210 case ShenandoahVerifier::_verify_forwarded_none: { 211 verify(ShenandoahAsserts::_safe_all, obj, oopDesc::unsafe_equals(obj, fwd), 212 "Should not be forwarded"); 213 break; 214 } 215 case ShenandoahVerifier::_verify_forwarded_allow: { 216 if (!oopDesc::unsafe_equals(obj, fwd)) { 217 verify(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg, 218 "Forwardee should be in another region"); 219 } 220 break; 221 } 222 default: 223 assert(false, "Unhandled forwarding verification"); 224 } 225 226 switch (_options._verify_cset) { 227 case ShenandoahVerifier::_verify_cset_disable: 228 // skip 229 break; 230 case ShenandoahVerifier::_verify_cset_none: 231 verify(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj), 232 "Should not have references to collection set"); 233 break; 234 case ShenandoahVerifier::_verify_cset_forwarded: 235 if (_heap->in_collection_set(obj)) { 236 verify(ShenandoahAsserts::_safe_all, obj, !oopDesc::unsafe_equals(obj, fwd), 237 "Object in collection set, should have forwardee"); 238 } 239 break; 240 default: 241 assert(false, "Unhandled cset verification"); 242 } 243 } 244 245 public: 246 247 /** 248 * Verify object with known interior reference. 249 * @param p interior reference where the object is referenced from; can be off-heap 250 * @param obj verified object 251 */ 252 template <class T> 253 void verify_oop_at(T* p, oop obj) { 254 _interior_loc = p; 255 verify_oop(obj); 256 _interior_loc = NULL; 257 } 258 259 /** 260 * Verify object without known interior reference. 261 * Useful when picking up the object at known offset in heap, 262 * but without knowing what objects reference it. 263 * @param obj verified object 264 */ 265 void verify_oop_standalone(oop obj) { 266 _interior_loc = NULL; 267 verify_oop(obj); 268 _interior_loc = NULL; 269 } 270 271 /** 272 * Verify oop fields from this object. 273 * @param obj host object for verified fields 274 */ 275 void verify_oops_from(oop obj) { 276 _loc = obj; 277 obj->oop_iterate(this); 278 _loc = NULL; 279 } 280 281 282 void do_oop(oop* p) { do_oop_work(p); } 283 void do_oop(narrowOop* p) { do_oop_work(p); } 284 }; 285 286 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { 287 private: 288 size_t _used, _committed, _garbage; 289 public: 290 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {}; 291 292 bool heap_region_do(ShenandoahHeapRegion* r) { 293 _used += r->used(); 294 _garbage += r->garbage(); 295 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; 296 return false; 297 } 298 299 size_t used() { return _used; } 300 size_t committed() { return _committed; } 301 size_t garbage() { return _garbage; } 302 }; 303 304 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { 305 private: 306 ShenandoahHeap* _heap; 307 const char* _phase; 308 ShenandoahVerifier::VerifyRegions _regions; 309 public: 310 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) : 311 _heap(ShenandoahHeap::heap()), 312 _phase(phase), 313 _regions(regions) {}; 314 315 void print_failure(ShenandoahHeapRegion* r, const char* label) { 316 ResourceMark rm; 317 318 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label); 319 320 stringStream ss; 321 r->print_on(&ss); 322 msg.append("%s", ss.as_string()); 323 324 report_vm_error(__FILE__, __LINE__, msg.buffer()); 325 } 326 327 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) { 328 if (!test) { 329 print_failure(r, msg); 330 } 331 } 332 333 bool heap_region_do(ShenandoahHeapRegion* r) { 334 switch (_regions) { 335 case ShenandoahVerifier::_verify_regions_disable: 336 break; 337 case ShenandoahVerifier::_verify_regions_notrash: 338 verify(r, !r->is_trash(), 339 "Should not have trash regions"); 340 break; 341 case ShenandoahVerifier::_verify_regions_nocset: 342 verify(r, !r->is_cset(), 343 "Should not have cset regions"); 344 break; 345 case ShenandoahVerifier::_verify_regions_notrash_nocset: 346 verify(r, !r->is_trash(), 347 "Should not have trash regions"); 348 verify(r, !r->is_cset(), 349 "Should not have cset regions"); 350 break; 351 default: 352 ShouldNotReachHere(); 353 } 354 355 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(), 356 "Capacity should match region size"); 357 358 verify(r, r->bottom() <= _heap->complete_marking_context()->top_at_mark_start(r->region_number()), 359 "Region top should not be less than bottom"); 360 361 verify(r, _heap->complete_marking_context()->top_at_mark_start(r->region_number()) <= r->top(), 362 "Complete TAMS should not be larger than top"); 363 364 verify(r, r->get_live_data_bytes() <= r->capacity(), 365 "Live data cannot be larger than capacity"); 366 367 verify(r, r->garbage() <= r->capacity(), 368 "Garbage cannot be larger than capacity"); 369 370 verify(r, r->used() <= r->capacity(), 371 "Used cannot be larger than capacity"); 372 373 verify(r, r->get_shared_allocs() <= r->capacity(), 374 "Shared alloc count should not be larger than capacity"); 375 376 verify(r, r->get_tlab_allocs() <= r->capacity(), 377 "TLAB alloc count should not be larger than capacity"); 378 379 verify(r, r->get_gclab_allocs() <= r->capacity(), 380 "GCLAB alloc count should not be larger than capacity"); 381 382 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(), 383 "Accurate accounting: shared + TLAB + GCLAB = used"); 384 385 verify(r, !r->is_empty() || !r->has_live(), 386 "Empty regions should not have live data"); 387 388 verify(r, r->is_cset() == r->in_collection_set(), 389 "Transitional: region flags and collection set agree"); 390 391 return false; 392 } 393 }; 394 395 class ShenandoahVerifierReachableTask : public AbstractGangTask { 396 private: 397 const char* _label; 398 ShenandoahRootProcessor* _rp; 399 ShenandoahVerifier::VerifyOptions _options; 400 ShenandoahHeap* _heap; 401 ShenandoahLivenessData* _ld; 402 MarkBitMap* _bitmap; 403 volatile jlong _processed; 404 405 public: 406 ShenandoahVerifierReachableTask(MarkBitMap* bitmap, 407 ShenandoahLivenessData* ld, 408 ShenandoahRootProcessor* rp, 409 const char* label, 410 ShenandoahVerifier::VerifyOptions options) : 411 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"), 412 _label(label), 413 _rp(rp), 414 _options(options), 415 _heap(ShenandoahHeap::heap()), 416 _ld(ld), 417 _bitmap(bitmap), 418 _processed(0) {}; 419 420 size_t processed() { 421 return (size_t) _processed; 422 } 423 424 virtual void work(uint worker_id) { 425 ResourceMark rm; 426 ShenandoahVerifierStack stack; 427 428 // On level 2, we need to only check the roots once. 429 // On level 3, we want to check the roots, and seed the local stack. 430 // It is a lesser evil to accept multiple root scans at level 3, because 431 // extended parallelism would buy us out. 432 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0)) 433 || (ShenandoahVerifyLevel >= 3)) { 434 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 435 ShenandoahMessageBuffer("Shenandoah verification failed; %s, Roots", _label), 436 _options); 437 _rp->process_all_roots_slow(&cl); 438 } 439 440 jlong processed = 0; 441 442 if (ShenandoahVerifyLevel >= 3) { 443 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 444 ShenandoahMessageBuffer("Shenandoah verification failed; %s, Reachable", _label), 445 _options); 446 while (!stack.is_empty()) { 447 processed++; 448 ShenandoahVerifierTask task = stack.pop(); 449 cl.verify_oops_from(task.obj()); 450 } 451 } 452 453 Atomic::add(processed, &_processed); 454 } 455 }; 456 457 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { 458 private: 459 const char* _label; 460 ShenandoahVerifier::VerifyOptions _options; 461 ShenandoahHeap *_heap; 462 ShenandoahLivenessData* _ld; 463 MarkBitMap* _bitmap; 464 volatile jint _claimed; 465 volatile jlong _processed; 466 467 public: 468 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap, 469 ShenandoahLivenessData* ld, 470 const char* label, 471 ShenandoahVerifier::VerifyOptions options) : 472 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"), 473 _label(label), 474 _options(options), 475 _heap(ShenandoahHeap::heap()), 476 _bitmap(bitmap), 477 _ld(ld), 478 _claimed(0), 479 _processed(0) {}; 480 481 size_t processed() { 482 return (size_t) _processed; 483 } 484 485 virtual void work(uint worker_id) { 486 ShenandoahVerifierStack stack; 487 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 488 ShenandoahMessageBuffer("Shenandoah verification failed; %s, Marked", _label), 489 _options); 490 assert((size_t)max_jint >= _heap->num_regions(), "Too many regions"); 491 while (true) { 492 size_t v = (size_t) (Atomic::add(1, &_claimed) - 1); 493 if (v < _heap->num_regions()) { 494 ShenandoahHeapRegion* r = _heap->get_region(v); 495 if (!r->is_humongous() && !r->is_trash()) { 496 work_regular(r, stack, cl); 497 } else if (r->is_humongous_start()) { 498 work_humongous(r, stack, cl); 499 } 500 } else { 501 break; 502 } 503 } 504 } 505 506 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { 507 jlong processed = 0; 508 HeapWord* obj = r->bottom() + BrooksPointer::word_size(); 509 if (_heap->complete_marking_context()->is_marked((oop)obj)) { 510 verify_and_follow(obj, stack, cl, &processed); 511 } 512 Atomic::add(processed, &_processed); 513 } 514 515 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { 516 jlong processed = 0; 517 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map(); 518 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r->region_number()); 519 520 // Bitmaps, before TAMS 521 if (tams > r->bottom()) { 522 HeapWord* start = r->bottom() + BrooksPointer::word_size(); 523 HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams); 524 525 while (addr < tams) { 526 verify_and_follow(addr, stack, cl, &processed); 527 addr += BrooksPointer::word_size(); 528 if (addr < tams) { 529 addr = mark_bit_map->getNextMarkedWordAddress(addr, tams); 530 } 531 } 532 } 533 534 // Size-based, after TAMS 535 { 536 HeapWord* limit = r->top(); 537 HeapWord* addr = tams + BrooksPointer::word_size(); 538 539 while (addr < limit) { 540 verify_and_follow(addr, stack, cl, &processed); 541 addr += oop(addr)->size() + BrooksPointer::word_size(); 542 } 543 } 544 545 Atomic::add(processed, &_processed); 546 } 547 548 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, jlong *processed) { 549 if (!_bitmap->parMark(addr)) return; 550 551 // Verify the object itself: 552 oop obj = oop(addr); 553 cl.verify_oop_standalone(obj); 554 555 // Verify everything reachable from that object too, hopefully realizing 556 // everything was already marked, and never touching further: 557 cl.verify_oops_from(obj); 558 (*processed)++; 559 560 while (!stack.is_empty()) { 561 ShenandoahVerifierTask task = stack.pop(); 562 cl.verify_oops_from(task.obj()); 563 (*processed)++; 564 } 565 } 566 }; 567 568 class VerifyThreadGCState : public ThreadClosure { 569 private: 570 const char* _label; 571 char _expected; 572 573 public: 574 VerifyThreadGCState(const char* label, char expected) : _expected(expected) {} 575 void do_thread(Thread* t) { 576 assert(t->is_Java_thread(), "sanity"); 577 char actual = ((JavaThread*)t)->gc_state(); 578 if (actual != _expected) { 579 fatal(err_msg("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual)); 580 } 581 } 582 }; 583 584 585 void ShenandoahVerifier::verify_at_safepoint(const char *label, 586 VerifyForwarded forwarded, VerifyMarked marked, 587 VerifyCollectionSet cset, 588 VerifyLiveness liveness, VerifyRegions regions, 589 VerifyGCState gcstate) { 590 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens"); 591 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize"); 592 593 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check 594 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/); 595 596 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel); 597 598 // GC state checks 599 { 600 char expected = -1; 601 bool enabled; 602 switch (gcstate) { 603 case _verify_gcstate_disable: 604 enabled = false; 605 break; 606 case _verify_gcstate_forwarded: 607 enabled = true; 608 expected = ShenandoahHeap::HAS_FORWARDED; 609 break; 610 case _verify_gcstate_stable: 611 enabled = true; 612 expected = ShenandoahHeap::STABLE; 613 break; 614 default: 615 enabled = false; 616 assert(false, "Unhandled gc-state verification"); 617 } 618 619 if (enabled) { 620 char actual = _heap->gc_state(); 621 if (actual != expected) { 622 fatal(err_msg("%s: Global gc-state: expected %d, actual %d", label, expected, actual)); 623 } 624 625 VerifyThreadGCState vtgcs(label, expected); 626 Threads::java_threads_do(&vtgcs); 627 } 628 } 629 630 // Heap size checks 631 { 632 ShenandoahHeapLocker lock(_heap->lock()); 633 634 ShenandoahCalculateRegionStatsClosure cl; 635 _heap->heap_region_iterate(&cl); 636 size_t heap_used = _heap->used(); 637 guarantee(cl.used() == heap_used, 638 err_msg("%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "K, regions-used = " SIZE_FORMAT "K", 639 label, heap_used/K, cl.used()/K)); 640 641 size_t heap_committed = _heap->committed(); 642 guarantee(cl.committed() == heap_committed, 643 err_msg("%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "K, regions-committed = " SIZE_FORMAT "K", 644 label, heap_committed/K, cl.committed()/K)); 645 } 646 647 // Internal heap region checks 648 if (ShenandoahVerifyLevel >= 1) { 649 ShenandoahVerifyHeapRegionClosure cl(label, regions); 650 _heap->heap_region_iterate(&cl, 651 /* skip_cset = */ false, 652 /* skip_humongous_cont = */ false); 653 } 654 655 OrderAccess::fence(); 656 _heap->make_parsable(false); 657 658 // Allocate temporary bitmap for storing marking wavefront: 659 MemRegion mr = MemRegion(_verification_bit_map->startWord(), _verification_bit_map->endWord()); 660 _verification_bit_map->clear_range_large(mr); 661 662 // Allocate temporary array for storing liveness data 663 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC); 664 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0); 665 666 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, cset, liveness, regions, gcstate); 667 668 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap. 669 // This verifies what application can see, since it only cares about reachable objects. 670 size_t count_reachable = 0; 671 if (ShenandoahVerifyLevel >= 2) { 672 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), 673 ShenandoahPhaseTimings::_num_phases); // no need for stats 674 675 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options); 676 _heap->workers()->run_task(&task); 677 count_reachable = task.processed(); 678 } 679 680 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector, 681 // not the application, can see during the region scans. There is no reason to process the objects 682 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS: 683 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics 684 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect) 685 // version 686 687 size_t count_marked = 0; 688 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) { 689 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options); 690 _heap->workers()->run_task(&task); 691 count_marked = task.processed(); 692 } else { 693 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_next || marked == _verify_marked_disable, "Should be"); 694 } 695 696 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes 697 // marked objects. 698 699 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) { 700 for (size_t i = 0; i < _heap->num_regions(); i++) { 701 ShenandoahHeapRegion* r = _heap->get_region(i); 702 703 jint verf_live = 0; 704 if (r->is_humongous()) { 705 // For humongous objects, test if start region is marked live, and if so, 706 // all humongous regions in that chain have live data equal to their "used". 707 jint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]); 708 if (start_live > 0) { 709 verf_live = (jint)(r->used() / HeapWordSize); 710 } 711 } else { 712 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]); 713 } 714 715 size_t reg_live = r->get_live_data_words(); 716 if (reg_live != (size_t)verf_live) { 717 ResourceMark rm; 718 stringStream ss; 719 r->print_on(&ss); 720 fatal(err_msg("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " INT32_FORMAT "\n%s", 721 label, reg_live, verf_live, ss.as_string())); 722 } 723 } 724 } 725 726 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)", 727 label, ShenandoahVerifyLevel, count_reachable, count_marked); 728 729 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld, mtGC); 730 } 731 732 void ShenandoahVerifier::verify_generic(VerifyOption vo) { 733 verify_at_safepoint( 734 "Generic Verification", 735 _verify_forwarded_allow, // conservatively allow forwarded 736 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 737 _verify_cset_disable, // cset may be inconsistent 738 _verify_liveness_disable, // no reliable liveness data 739 _verify_regions_disable, // no reliable region data 740 _verify_gcstate_disable // no data about gcstate 741 ); 742 } 743 744 void ShenandoahVerifier::verify_before_concmark() { 745 if (_heap->has_forwarded_objects()) { 746 verify_at_safepoint( 747 "Before Mark", 748 _verify_forwarded_allow, // may have forwarded references 749 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 750 _verify_cset_forwarded, // allow forwarded references to cset 751 _verify_liveness_disable, // no reliable liveness data 752 _verify_regions_notrash, // no trash regions 753 _verify_gcstate_forwarded // there are forwarded objects 754 ); 755 } else { 756 verify_at_safepoint( 757 "Before Mark", 758 _verify_forwarded_none, // UR should have fixed up 759 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 760 _verify_cset_none, // UR should have fixed this 761 _verify_liveness_disable, // no reliable liveness data 762 _verify_regions_notrash, // no trash regions 763 _verify_gcstate_stable // there are no forwarded objects 764 ); 765 } 766 } 767 768 void ShenandoahVerifier::verify_after_concmark() { 769 verify_at_safepoint( 770 "After Mark", 771 _verify_forwarded_none, // no forwarded references 772 _verify_marked_complete, // bitmaps as precise as we can get 773 _verify_cset_none, // no references to cset anymore 774 _verify_liveness_complete, // liveness data must be complete here 775 _verify_regions_disable, // trash regions not yet recycled 776 _verify_gcstate_stable // mark should have stabilized the heap 777 ); 778 } 779 780 void ShenandoahVerifier::verify_before_evacuation() { 781 // Evacuation is always preceded by mark, but we want to have a sanity check after 782 // selecting the collection set, and (immediate) regions recycling 783 verify_at_safepoint( 784 "Before Evacuation", 785 _verify_forwarded_none, // no forwarded references 786 _verify_marked_complete, // walk over marked objects too 787 _verify_cset_disable, // skip, verified after mark 788 _verify_liveness_disable, // skip, verified after mark 789 _verify_regions_disable, // trash regions not yet recycled 790 _verify_gcstate_stable // mark should have stabilized the heap 791 ); 792 } 793 794 void ShenandoahVerifier::verify_after_evacuation() { 795 verify_at_safepoint( 796 "After Evacuation", 797 _verify_forwarded_allow, // objects are still forwarded 798 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 799 _verify_cset_forwarded, // all cset refs are fully forwarded 800 _verify_liveness_disable, // no reliable liveness data anymore 801 _verify_regions_notrash, // trash regions have been recycled already 802 _verify_gcstate_forwarded // evacuation produced some forwarded objects 803 ); 804 } 805 806 void ShenandoahVerifier::verify_before_updaterefs() { 807 verify_at_safepoint( 808 "Before Updating References", 809 _verify_forwarded_allow, // forwarded references allowed 810 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 811 _verify_cset_forwarded, // all cset refs are fully forwarded 812 _verify_liveness_disable, // no reliable liveness data anymore 813 _verify_regions_notrash, // trash regions have been recycled already 814 _verify_gcstate_forwarded // evacuation should have produced some forwarded objects 815 ); 816 } 817 818 void ShenandoahVerifier::verify_after_updaterefs() { 819 verify_at_safepoint( 820 "After Updating References", 821 _verify_forwarded_none, // no forwarded references 822 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 823 _verify_cset_none, // no cset references, all updated 824 _verify_liveness_disable, // no reliable liveness data anymore 825 _verify_regions_nocset, // no cset regions, trash regions have appeared 826 _verify_gcstate_stable // update refs had cleaned up forwarded objects 827 ); 828 } 829 830 void ShenandoahVerifier::verify_before_fullgc() { 831 verify_at_safepoint( 832 "Before Full GC", 833 _verify_forwarded_allow, // can have forwarded objects 834 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 835 _verify_cset_disable, // cset might be foobared 836 _verify_liveness_disable, // no reliable liveness data anymore 837 _verify_regions_disable, // no reliable region data here 838 _verify_gcstate_disable // no reliable gcstate data 839 ); 840 } 841 842 void ShenandoahVerifier::verify_after_fullgc() { 843 verify_at_safepoint( 844 "After Full GC", 845 _verify_forwarded_none, // all objects are non-forwarded 846 _verify_marked_complete, // all objects are marked in complete bitmap 847 _verify_cset_none, // no cset references 848 _verify_liveness_disable, // no reliable liveness data anymore 849 _verify_regions_notrash_nocset, // no trash, no cset 850 _verify_gcstate_stable // degenerated refs had cleaned up forwarded objects 851 ); 852 } 853 854 void ShenandoahVerifier::verify_after_degenerated() { 855 verify_at_safepoint( 856 "After Degenerated GC", 857 _verify_forwarded_none, // all objects are non-forwarded 858 _verify_marked_complete, // all objects are marked in complete bitmap 859 _verify_cset_none, // no cset references 860 _verify_liveness_disable, // no reliable liveness data anymore 861 _verify_regions_notrash_nocset, // no trash, no cset 862 _verify_gcstate_stable // full gc cleaned up everything 863 ); 864 }