1 /* 2 * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shenandoah/shenandoahAsserts.hpp" 28 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp" 29 #include "gc/shenandoah/shenandoahForwarding.inline.hpp" 30 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" 33 #include "gc/shenandoah/shenandoahRootProcessor.hpp" 34 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" 35 #include "gc/shenandoah/shenandoahUtils.hpp" 36 #include "gc/shenandoah/shenandoahVerifier.hpp" 37 #include "memory/allocation.hpp" 38 #include "memory/iterator.inline.hpp" 39 #include "memory/resourceArea.hpp" 40 #include "oops/compressedOops.inline.hpp" 41 #include "runtime/atomic.hpp" 42 #include "runtime/orderAccess.hpp" 43 #include "utilities/align.hpp" 44 45 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp) 46 #ifdef verify_oop 47 #undef verify_oop 48 #endif 49 50 class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { 51 private: 52 const char* _phase; 53 ShenandoahVerifier::VerifyOptions _options; 54 ShenandoahVerifierStack* _stack; 55 ShenandoahHeap* _heap; 56 MarkBitMap* _map; 57 ShenandoahLivenessData* _ld; 58 void* _interior_loc; 59 oop _loc; 60 61 public: 62 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld, 63 const char* phase, ShenandoahVerifier::VerifyOptions options) : 64 _phase(phase), 65 _options(options), 66 _stack(stack), 67 _heap(ShenandoahHeap::heap()), 68 _map(map), 69 _ld(ld), 70 _interior_loc(NULL), 71 _loc(NULL) { } 72 73 private: 74 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) { 75 if (!test) { 76 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__); 77 } 78 } 79 80 template <class T> 81 void do_oop_work(T* p) { 82 T o = RawAccess<>::oop_load(p); 83 if (!CompressedOops::is_null(o)) { 84 oop obj = CompressedOops::decode_not_null(o); 85 86 // Single threaded verification can use faster non-atomic stack and bitmap 87 // methods. 88 // 89 // For performance reasons, only fully verify non-marked field values. 90 // We are here when the host object for *p is already marked. 91 92 if (_map->par_mark(obj)) { 93 verify_oop_at(p, obj); 94 _stack->push(ShenandoahVerifierTask(obj)); 95 } 96 } 97 } 98 99 void verify_oop(oop obj) { 100 // Perform consistency checks with gradually decreasing safety level. This guarantees 101 // that failure report would not try to touch something that was not yet verified to be 102 // safe to process. 103 104 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj), 105 "oop must be in heap"); 106 check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj), 107 "oop must be aligned"); 108 109 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); 110 Klass* obj_klass = obj->klass_or_null(); 111 112 // Verify that obj is not in dead space: 113 { 114 // Do this before touching obj->size() 115 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL, 116 "Object klass pointer should not be NULL"); 117 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass), 118 "Object klass pointer must go to metaspace"); 119 120 HeapWord *obj_addr = cast_from_oop<HeapWord*>(obj); 121 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(), 122 "Object start should be within the region"); 123 124 if (!obj_reg->is_humongous()) { 125 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), 126 "Object end should be within the region"); 127 } else { 128 size_t humongous_start = obj_reg->index(); 129 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift()); 130 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) { 131 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(), 132 "Humongous object is in continuation that fits it"); 133 } 134 } 135 136 // ------------ obj is safe at this point -------------- 137 138 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(), 139 "Object should be in active region"); 140 141 switch (_options._verify_liveness) { 142 case ShenandoahVerifier::_verify_liveness_disable: 143 // skip 144 break; 145 case ShenandoahVerifier::_verify_liveness_complete: 146 Atomic::add(&_ld[obj_reg->index()], (uint) obj->size()); 147 // fallthrough for fast failure for un-live regions: 148 case ShenandoahVerifier::_verify_liveness_conservative: 149 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(), 150 "Object must belong to region with live data"); 151 break; 152 default: 153 assert(false, "Unhandled liveness verification"); 154 } 155 } 156 157 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj); 158 159 ShenandoahHeapRegion* fwd_reg = NULL; 160 161 if (obj != fwd) { 162 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd), 163 "Forwardee must be in heap"); 164 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd), 165 "Forwardee is set"); 166 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd), 167 "Forwardee must be aligned"); 168 169 // Do this before touching fwd->size() 170 Klass* fwd_klass = fwd->klass_or_null(); 171 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL, 172 "Forwardee klass pointer should not be NULL"); 173 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass), 174 "Forwardee klass pointer must go to metaspace"); 175 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass, 176 "Forwardee klass pointer must go to metaspace"); 177 178 fwd_reg = _heap->heap_region_containing(fwd); 179 180 // Verify that forwardee is not in the dead space: 181 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(), 182 "Should have no humongous forwardees"); 183 184 HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd); 185 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), 186 "Forwardee start should be within the region"); 187 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), 188 "Forwardee end should be within the region"); 189 190 oop fwd2 = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(fwd); 191 check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2), 192 "Double forwarding"); 193 } else { 194 fwd_reg = obj_reg; 195 } 196 197 // ------------ obj and fwd are safe at this point -------------- 198 199 switch (_options._verify_marked) { 200 case ShenandoahVerifier::_verify_marked_disable: 201 // skip 202 break; 203 case ShenandoahVerifier::_verify_marked_incomplete: 204 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj), 205 "Must be marked in incomplete bitmap"); 206 break; 207 case ShenandoahVerifier::_verify_marked_complete: 208 check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj), 209 "Must be marked in complete bitmap"); 210 break; 211 default: 212 assert(false, "Unhandled mark verification"); 213 } 214 215 switch (_options._verify_forwarded) { 216 case ShenandoahVerifier::_verify_forwarded_disable: 217 // skip 218 break; 219 case ShenandoahVerifier::_verify_forwarded_none: { 220 check(ShenandoahAsserts::_safe_all, obj, (obj == fwd), 221 "Should not be forwarded"); 222 break; 223 } 224 case ShenandoahVerifier::_verify_forwarded_allow: { 225 if (obj != fwd) { 226 check(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg, 227 "Forwardee should be in another region"); 228 } 229 break; 230 } 231 default: 232 assert(false, "Unhandled forwarding verification"); 233 } 234 235 switch (_options._verify_cset) { 236 case ShenandoahVerifier::_verify_cset_disable: 237 // skip 238 break; 239 case ShenandoahVerifier::_verify_cset_none: 240 check(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj), 241 "Should not have references to collection set"); 242 break; 243 case ShenandoahVerifier::_verify_cset_forwarded: 244 if (_heap->in_collection_set(obj)) { 245 check(ShenandoahAsserts::_safe_all, obj, (obj != fwd), 246 "Object in collection set, should have forwardee"); 247 } 248 break; 249 default: 250 assert(false, "Unhandled cset verification"); 251 } 252 253 } 254 255 public: 256 /** 257 * Verify object with known interior reference. 258 * @param p interior reference where the object is referenced from; can be off-heap 259 * @param obj verified object 260 */ 261 template <class T> 262 void verify_oop_at(T* p, oop obj) { 263 _interior_loc = p; 264 verify_oop(obj); 265 _interior_loc = NULL; 266 } 267 268 /** 269 * Verify object without known interior reference. 270 * Useful when picking up the object at known offset in heap, 271 * but without knowing what objects reference it. 272 * @param obj verified object 273 */ 274 void verify_oop_standalone(oop obj) { 275 _interior_loc = NULL; 276 verify_oop(obj); 277 _interior_loc = NULL; 278 } 279 280 /** 281 * Verify oop fields from this object. 282 * @param obj host object for verified fields 283 */ 284 void verify_oops_from(oop obj) { 285 _loc = obj; 286 obj->oop_iterate(this); 287 _loc = NULL; 288 } 289 290 virtual void do_oop(oop* p) { do_oop_work(p); } 291 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 292 }; 293 294 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { 295 private: 296 size_t _used, _committed, _garbage; 297 public: 298 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {}; 299 300 void heap_region_do(ShenandoahHeapRegion* r) { 301 _used += r->used(); 302 _garbage += r->garbage(); 303 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; 304 } 305 306 size_t used() { return _used; } 307 size_t committed() { return _committed; } 308 size_t garbage() { return _garbage; } 309 }; 310 311 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { 312 private: 313 ShenandoahHeap* _heap; 314 const char* _phase; 315 ShenandoahVerifier::VerifyRegions _regions; 316 public: 317 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) : 318 _heap(ShenandoahHeap::heap()), 319 _phase(phase), 320 _regions(regions) {}; 321 322 void print_failure(ShenandoahHeapRegion* r, const char* label) { 323 ResourceMark rm; 324 325 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label); 326 327 stringStream ss; 328 r->print_on(&ss); 329 msg.append("%s", ss.as_string()); 330 331 report_vm_error(__FILE__, __LINE__, msg.buffer()); 332 } 333 334 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) { 335 if (!test) { 336 print_failure(r, msg); 337 } 338 } 339 340 void heap_region_do(ShenandoahHeapRegion* r) { 341 switch (_regions) { 342 case ShenandoahVerifier::_verify_regions_disable: 343 break; 344 case ShenandoahVerifier::_verify_regions_notrash: 345 verify(r, !r->is_trash(), 346 "Should not have trash regions"); 347 break; 348 case ShenandoahVerifier::_verify_regions_nocset: 349 verify(r, !r->is_cset(), 350 "Should not have cset regions"); 351 break; 352 case ShenandoahVerifier::_verify_regions_notrash_nocset: 353 verify(r, !r->is_trash(), 354 "Should not have trash regions"); 355 verify(r, !r->is_cset(), 356 "Should not have cset regions"); 357 break; 358 default: 359 ShouldNotReachHere(); 360 } 361 362 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(), 363 "Capacity should match region size"); 364 365 verify(r, r->bottom() <= r->top(), 366 "Region top should not be less than bottom"); 367 368 verify(r, r->bottom() <= _heap->marking_context()->top_at_mark_start(r), 369 "Region TAMS should not be less than bottom"); 370 371 verify(r, _heap->marking_context()->top_at_mark_start(r) <= r->top(), 372 "Complete TAMS should not be larger than top"); 373 374 verify(r, r->get_live_data_bytes() <= r->capacity(), 375 "Live data cannot be larger than capacity"); 376 377 verify(r, r->garbage() <= r->capacity(), 378 "Garbage cannot be larger than capacity"); 379 380 verify(r, r->used() <= r->capacity(), 381 "Used cannot be larger than capacity"); 382 383 verify(r, r->get_shared_allocs() <= r->capacity(), 384 "Shared alloc count should not be larger than capacity"); 385 386 verify(r, r->get_tlab_allocs() <= r->capacity(), 387 "TLAB alloc count should not be larger than capacity"); 388 389 verify(r, r->get_gclab_allocs() <= r->capacity(), 390 "GCLAB alloc count should not be larger than capacity"); 391 392 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(), 393 "Accurate accounting: shared + TLAB + GCLAB = used"); 394 395 verify(r, !r->is_empty() || !r->has_live(), 396 "Empty regions should not have live data"); 397 398 verify(r, r->is_cset() == _heap->collection_set()->is_in(r), 399 "Transitional: region flags and collection set agree"); 400 } 401 }; 402 403 class ShenandoahVerifierReachableTask : public AbstractGangTask { 404 private: 405 const char* _label; 406 ShenandoahRootVerifier* _verifier; 407 ShenandoahVerifier::VerifyOptions _options; 408 ShenandoahHeap* _heap; 409 ShenandoahLivenessData* _ld; 410 MarkBitMap* _bitmap; 411 volatile size_t _processed; 412 413 public: 414 ShenandoahVerifierReachableTask(MarkBitMap* bitmap, 415 ShenandoahLivenessData* ld, 416 ShenandoahRootVerifier* verifier, 417 const char* label, 418 ShenandoahVerifier::VerifyOptions options) : 419 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"), 420 _label(label), 421 _verifier(verifier), 422 _options(options), 423 _heap(ShenandoahHeap::heap()), 424 _ld(ld), 425 _bitmap(bitmap), 426 _processed(0) {}; 427 428 size_t processed() { 429 return _processed; 430 } 431 432 virtual void work(uint worker_id) { 433 ResourceMark rm; 434 ShenandoahVerifierStack stack; 435 436 // On level 2, we need to only check the roots once. 437 // On level 3, we want to check the roots, and seed the local stack. 438 // It is a lesser evil to accept multiple root scans at level 3, because 439 // extended parallelism would buy us out. 440 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0)) 441 || (ShenandoahVerifyLevel >= 3)) { 442 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 443 ShenandoahMessageBuffer("%s, Roots", _label), 444 _options); 445 if (_heap->unload_classes()) { 446 _verifier->strong_roots_do(&cl); 447 } else { 448 _verifier->roots_do(&cl); 449 } 450 } 451 452 size_t processed = 0; 453 454 if (ShenandoahVerifyLevel >= 3) { 455 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 456 ShenandoahMessageBuffer("%s, Reachable", _label), 457 _options); 458 while (!stack.is_empty()) { 459 processed++; 460 ShenandoahVerifierTask task = stack.pop(); 461 cl.verify_oops_from(task.obj()); 462 } 463 } 464 465 Atomic::add(&_processed, processed); 466 } 467 }; 468 469 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { 470 private: 471 const char* _label; 472 ShenandoahVerifier::VerifyOptions _options; 473 ShenandoahHeap *_heap; 474 MarkBitMap* _bitmap; 475 ShenandoahLivenessData* _ld; 476 volatile size_t _claimed; 477 volatile size_t _processed; 478 479 public: 480 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap, 481 ShenandoahLivenessData* ld, 482 const char* label, 483 ShenandoahVerifier::VerifyOptions options) : 484 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"), 485 _label(label), 486 _options(options), 487 _heap(ShenandoahHeap::heap()), 488 _bitmap(bitmap), 489 _ld(ld), 490 _claimed(0), 491 _processed(0) {}; 492 493 size_t processed() { 494 return _processed; 495 } 496 497 virtual void work(uint worker_id) { 498 ShenandoahVerifierStack stack; 499 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 500 ShenandoahMessageBuffer("%s, Marked", _label), 501 _options); 502 503 while (true) { 504 size_t v = Atomic::fetch_and_add(&_claimed, 1u); 505 if (v < _heap->num_regions()) { 506 ShenandoahHeapRegion* r = _heap->get_region(v); 507 if (!r->is_humongous() && !r->is_trash()) { 508 work_regular(r, stack, cl); 509 } else if (r->is_humongous_start()) { 510 work_humongous(r, stack, cl); 511 } 512 } else { 513 break; 514 } 515 } 516 } 517 518 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { 519 size_t processed = 0; 520 HeapWord* obj = r->bottom(); 521 if (_heap->complete_marking_context()->is_marked((oop)obj)) { 522 verify_and_follow(obj, stack, cl, &processed); 523 } 524 Atomic::add(&_processed, processed); 525 } 526 527 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { 528 size_t processed = 0; 529 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map(); 530 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r); 531 532 // Bitmaps, before TAMS 533 if (tams > r->bottom()) { 534 HeapWord* start = r->bottom(); 535 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams); 536 537 while (addr < tams) { 538 verify_and_follow(addr, stack, cl, &processed); 539 addr += 1; 540 if (addr < tams) { 541 addr = mark_bit_map->get_next_marked_addr(addr, tams); 542 } 543 } 544 } 545 546 // Size-based, after TAMS 547 { 548 HeapWord* limit = r->top(); 549 HeapWord* addr = tams; 550 551 while (addr < limit) { 552 verify_and_follow(addr, stack, cl, &processed); 553 addr += oop(addr)->size(); 554 } 555 } 556 557 Atomic::add(&_processed, processed); 558 } 559 560 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) { 561 if (!_bitmap->par_mark(addr)) return; 562 563 // Verify the object itself: 564 oop obj = oop(addr); 565 cl.verify_oop_standalone(obj); 566 567 // Verify everything reachable from that object too, hopefully realizing 568 // everything was already marked, and never touching further: 569 cl.verify_oops_from(obj); 570 (*processed)++; 571 572 while (!stack.is_empty()) { 573 ShenandoahVerifierTask task = stack.pop(); 574 cl.verify_oops_from(task.obj()); 575 (*processed)++; 576 } 577 } 578 }; 579 580 class VerifyThreadGCState : public ThreadClosure { 581 private: 582 const char* _label; 583 char _expected; 584 585 public: 586 VerifyThreadGCState(const char* label, char expected) : _expected(expected) {} 587 void do_thread(Thread* t) { 588 char actual = ShenandoahThreadLocalData::gc_state(t); 589 if (actual != _expected) { 590 fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual); 591 } 592 } 593 }; 594 595 class ShenandoahGCStateResetter : public StackObj { 596 private: 597 ShenandoahHeap* const _heap; 598 char _gc_state; 599 600 public: 601 ShenandoahGCStateResetter() : _heap(ShenandoahHeap::heap()) { 602 _gc_state = _heap->gc_state(); 603 _heap->_gc_state.clear(); 604 } 605 606 ~ShenandoahGCStateResetter() { 607 _heap->_gc_state.set(_gc_state); 608 assert(_heap->gc_state() == _gc_state, "Should be restored"); 609 } 610 }; 611 612 void ShenandoahVerifier::verify_at_safepoint(const char *label, 613 VerifyForwarded forwarded, VerifyMarked marked, 614 VerifyCollectionSet cset, 615 VerifyLiveness liveness, VerifyRegions regions, 616 VerifyGCState gcstate, 617 VerifyWeakRoots weak_roots) { 618 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens"); 619 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize"); 620 621 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check 622 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/); 623 624 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel); 625 626 // GC state checks 627 { 628 char expected = -1; 629 bool enabled; 630 switch (gcstate) { 631 case _verify_gcstate_disable: 632 enabled = false; 633 break; 634 case _verify_gcstate_forwarded: 635 enabled = true; 636 expected = ShenandoahHeap::HAS_FORWARDED; 637 break; 638 case _verify_gcstate_evacuation: 639 enabled = true; 640 expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION; 641 break; 642 case _verify_gcstate_stable: 643 enabled = true; 644 expected = ShenandoahHeap::STABLE; 645 break; 646 default: 647 enabled = false; 648 assert(false, "Unhandled gc-state verification"); 649 } 650 651 if (enabled) { 652 char actual = _heap->gc_state(); 653 if (actual != expected) { 654 fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual); 655 } 656 657 VerifyThreadGCState vtgcs(label, expected); 658 Threads::java_threads_do(&vtgcs); 659 } 660 } 661 662 // Deactivate barriers temporarily: Verifier wants plain heap accesses 663 ShenandoahGCStateResetter resetter; 664 665 // Heap size checks 666 { 667 ShenandoahHeapLocker lock(_heap->lock()); 668 669 ShenandoahCalculateRegionStatsClosure cl; 670 _heap->heap_region_iterate(&cl); 671 size_t heap_used = _heap->used(); 672 guarantee(cl.used() == heap_used, 673 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", 674 label, 675 byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used), 676 byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used())); 677 678 size_t heap_committed = _heap->committed(); 679 guarantee(cl.committed() == heap_committed, 680 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "%s, regions-committed = " SIZE_FORMAT "%s", 681 label, 682 byte_size_in_proper_unit(heap_committed), proper_unit_for_byte_size(heap_committed), 683 byte_size_in_proper_unit(cl.committed()), proper_unit_for_byte_size(cl.committed())); 684 } 685 686 // Internal heap region checks 687 if (ShenandoahVerifyLevel >= 1) { 688 ShenandoahVerifyHeapRegionClosure cl(label, regions); 689 _heap->heap_region_iterate(&cl); 690 } 691 692 OrderAccess::fence(); 693 694 if (UseTLAB) { 695 _heap->labs_make_parsable(); 696 } 697 698 // Allocate temporary bitmap for storing marking wavefront: 699 _verification_bit_map->clear(); 700 701 // Allocate temporary array for storing liveness data 702 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC); 703 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0); 704 705 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, cset, liveness, regions, gcstate); 706 707 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap. 708 // This verifies what application can see, since it only cares about reachable objects. 709 size_t count_reachable = 0; 710 if (ShenandoahVerifyLevel >= 2) { 711 ShenandoahRootVerifier verifier; 712 switch (weak_roots) { 713 case _verify_serial_weak_roots: 714 verifier.excludes(ShenandoahRootVerifier::ConcurrentWeakRoots); 715 break; 716 case _verify_concurrent_weak_roots: 717 verifier.excludes(ShenandoahRootVerifier::SerialWeakRoots); 718 break; 719 case _verify_all_weak_roots: 720 break; 721 default: 722 ShouldNotReachHere(); 723 } 724 725 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &verifier, label, options); 726 _heap->workers()->run_task(&task); 727 count_reachable = task.processed(); 728 } 729 730 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector, 731 // not the application, can see during the region scans. There is no reason to process the objects 732 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS: 733 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics 734 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect) 735 // version 736 737 size_t count_marked = 0; 738 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) { 739 guarantee(_heap->marking_context()->is_complete(), "Marking context should be complete"); 740 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options); 741 _heap->workers()->run_task(&task); 742 count_marked = task.processed(); 743 } else { 744 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_incomplete || marked == _verify_marked_disable, "Should be"); 745 } 746 747 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes 748 // marked objects. 749 750 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) { 751 for (size_t i = 0; i < _heap->num_regions(); i++) { 752 ShenandoahHeapRegion* r = _heap->get_region(i); 753 754 juint verf_live = 0; 755 if (r->is_humongous()) { 756 // For humongous objects, test if start region is marked live, and if so, 757 // all humongous regions in that chain have live data equal to their "used". 758 juint start_live = Atomic::load_acquire(&ld[r->humongous_start_region()->index()]); 759 if (start_live > 0) { 760 verf_live = (juint)(r->used() / HeapWordSize); 761 } 762 } else { 763 verf_live = Atomic::load_acquire(&ld[r->index()]); 764 } 765 766 size_t reg_live = r->get_live_data_words(); 767 if (reg_live != verf_live) { 768 ResourceMark rm; 769 stringStream ss; 770 r->print_on(&ss); 771 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s", 772 label, reg_live, verf_live, ss.as_string()); 773 } 774 } 775 } 776 777 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)", 778 label, ShenandoahVerifyLevel, count_reachable, count_marked); 779 780 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld); 781 } 782 783 void ShenandoahVerifier::verify_generic(VerifyOption vo) { 784 verify_at_safepoint( 785 "Generic Verification", 786 _verify_forwarded_allow, // conservatively allow forwarded 787 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 788 _verify_cset_disable, // cset may be inconsistent 789 _verify_liveness_disable, // no reliable liveness data 790 _verify_regions_disable, // no reliable region data 791 _verify_gcstate_disable, // no data about gcstate 792 _verify_all_weak_roots 793 ); 794 } 795 796 void ShenandoahVerifier::verify_before_concmark() { 797 verify_at_safepoint( 798 "Before Mark", 799 _verify_forwarded_none, // UR should have fixed up 800 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 801 _verify_cset_none, // UR should have fixed this 802 _verify_liveness_disable, // no reliable liveness data 803 _verify_regions_notrash, // no trash regions 804 _verify_gcstate_stable, // there are no forwarded objects 805 _verify_all_weak_roots 806 ); 807 } 808 809 void ShenandoahVerifier::verify_after_concmark() { 810 verify_at_safepoint( 811 "After Mark", 812 _verify_forwarded_none, // no forwarded references 813 _verify_marked_complete, // bitmaps as precise as we can get 814 _verify_cset_none, // no references to cset anymore 815 _verify_liveness_complete, // liveness data must be complete here 816 _verify_regions_disable, // trash regions not yet recycled 817 _verify_gcstate_stable, // mark should have stabilized the heap 818 _verify_all_weak_roots 819 ); 820 } 821 822 void ShenandoahVerifier::verify_before_evacuation() { 823 // Concurrent weak roots are evacuated during concurrent phase 824 VerifyWeakRoots verify_weak_roots = ShenandoahConcurrentRoots::should_do_concurrent_class_unloading() ? 825 _verify_serial_weak_roots : 826 _verify_all_weak_roots; 827 828 verify_at_safepoint( 829 "Before Evacuation", 830 _verify_forwarded_none, // no forwarded references 831 _verify_marked_complete, // walk over marked objects too 832 _verify_cset_disable, // non-forwarded references to cset expected 833 _verify_liveness_complete, // liveness data must be complete here 834 _verify_regions_disable, // trash regions not yet recycled 835 _verify_gcstate_stable, // mark should have stabilized the heap 836 verify_weak_roots 837 ); 838 } 839 840 void ShenandoahVerifier::verify_during_evacuation() { 841 // Concurrent weak roots are evacuated during concurrent phase 842 VerifyWeakRoots verify_weak_roots = ShenandoahConcurrentRoots::should_do_concurrent_class_unloading() ? 843 _verify_serial_weak_roots : 844 _verify_all_weak_roots; 845 846 verify_at_safepoint( 847 "During Evacuation", 848 _verify_forwarded_allow, // some forwarded references are allowed 849 _verify_marked_disable, // walk only roots 850 _verify_cset_disable, // some cset references are not forwarded yet 851 _verify_liveness_disable, // liveness data might be already stale after pre-evacs 852 _verify_regions_disable, // trash regions not yet recycled 853 _verify_gcstate_evacuation, // evacuation is in progress 854 verify_weak_roots 855 ); 856 } 857 858 void ShenandoahVerifier::verify_after_evacuation() { 859 verify_at_safepoint( 860 "After Evacuation", 861 _verify_forwarded_allow, // objects are still forwarded 862 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 863 _verify_cset_forwarded, // all cset refs are fully forwarded 864 _verify_liveness_disable, // no reliable liveness data anymore 865 _verify_regions_notrash, // trash regions have been recycled already 866 _verify_gcstate_forwarded, // evacuation produced some forwarded objects 867 _verify_all_weak_roots 868 ); 869 } 870 871 void ShenandoahVerifier::verify_before_updaterefs() { 872 verify_at_safepoint( 873 "Before Updating References", 874 _verify_forwarded_allow, // forwarded references allowed 875 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 876 _verify_cset_forwarded, // all cset refs are fully forwarded 877 _verify_liveness_disable, // no reliable liveness data anymore 878 _verify_regions_notrash, // trash regions have been recycled already 879 _verify_gcstate_forwarded, // evacuation should have produced some forwarded objects 880 _verify_all_weak_roots 881 ); 882 } 883 884 void ShenandoahVerifier::verify_after_updaterefs() { 885 verify_at_safepoint( 886 "After Updating References", 887 _verify_forwarded_none, // no forwarded references 888 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 889 _verify_cset_none, // no cset references, all updated 890 _verify_liveness_disable, // no reliable liveness data anymore 891 _verify_regions_nocset, // no cset regions, trash regions have appeared 892 _verify_gcstate_stable, // update refs had cleaned up forwarded objects 893 _verify_all_weak_roots 894 ); 895 } 896 897 void ShenandoahVerifier::verify_after_degenerated() { 898 verify_at_safepoint( 899 "After Degenerated GC", 900 _verify_forwarded_none, // all objects are non-forwarded 901 _verify_marked_complete, // all objects are marked in complete bitmap 902 _verify_cset_none, // no cset references 903 _verify_liveness_disable, // no reliable liveness data anymore 904 _verify_regions_notrash_nocset, // no trash, no cset 905 _verify_gcstate_stable, // degenerated refs had cleaned up forwarded objects 906 _verify_all_weak_roots 907 ); 908 } 909 910 void ShenandoahVerifier::verify_before_fullgc() { 911 verify_at_safepoint( 912 "Before Full GC", 913 _verify_forwarded_allow, // can have forwarded objects 914 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 915 _verify_cset_disable, // cset might be foobared 916 _verify_liveness_disable, // no reliable liveness data anymore 917 _verify_regions_disable, // no reliable region data here 918 _verify_gcstate_disable, // no reliable gcstate data 919 _verify_all_weak_roots 920 ); 921 } 922 923 void ShenandoahVerifier::verify_after_fullgc() { 924 verify_at_safepoint( 925 "After Full GC", 926 _verify_forwarded_none, // all objects are non-forwarded 927 _verify_marked_complete, // all objects are marked in complete bitmap 928 _verify_cset_none, // no cset references 929 _verify_liveness_disable, // no reliable liveness data anymore 930 _verify_regions_notrash_nocset, // no trash, no cset 931 _verify_gcstate_stable, // full gc cleaned up everything 932 _verify_all_weak_roots 933 ); 934 } 935 936 class ShenandoahVerifyNoForwared : public OopClosure { 937 private: 938 template <class T> 939 void do_oop_work(T* p) { 940 T o = RawAccess<>::oop_load(p); 941 if (!CompressedOops::is_null(o)) { 942 oop obj = CompressedOops::decode_not_null(o); 943 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj); 944 if (obj != fwd) { 945 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, 946 "Verify Roots", "Should not be forwarded", __FILE__, __LINE__); 947 } 948 } 949 } 950 951 public: 952 void do_oop(narrowOop* p) { do_oop_work(p); } 953 void do_oop(oop* p) { do_oop_work(p); } 954 }; 955 956 class ShenandoahVerifyInToSpaceClosure : public OopClosure { 957 private: 958 template <class T> 959 void do_oop_work(T* p) { 960 T o = RawAccess<>::oop_load(p); 961 if (!CompressedOops::is_null(o)) { 962 oop obj = CompressedOops::decode_not_null(o); 963 ShenandoahHeap* heap = ShenandoahHeap::heap(); 964 965 if (!heap->marking_context()->is_marked(obj)) { 966 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, 967 "Verify Roots In To-Space", "Should be marked", __FILE__, __LINE__); 968 } 969 970 if (heap->in_collection_set(obj)) { 971 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, 972 "Verify Roots In To-Space", "Should not be in collection set", __FILE__, __LINE__); 973 } 974 975 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj); 976 if (obj != fwd) { 977 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, 978 "Verify Roots In To-Space", "Should not be forwarded", __FILE__, __LINE__); 979 } 980 } 981 } 982 983 public: 984 void do_oop(narrowOop* p) { do_oop_work(p); } 985 void do_oop(oop* p) { do_oop_work(p); } 986 }; 987 988 void ShenandoahVerifier::verify_roots_in_to_space() { 989 ShenandoahRootVerifier verifier; 990 ShenandoahVerifyInToSpaceClosure cl; 991 verifier.oops_do(&cl); 992 } 993 994 void ShenandoahVerifier::verify_roots_in_to_space_except(ShenandoahRootVerifier::RootTypes types) { 995 ShenandoahRootVerifier verifier; 996 verifier.excludes(types); 997 ShenandoahVerifyInToSpaceClosure cl; 998 verifier.oops_do(&cl); 999 } 1000 1001 void ShenandoahVerifier::verify_roots_no_forwarded() { 1002 ShenandoahRootVerifier verifier; 1003 ShenandoahVerifyNoForwared cl; 1004 verifier.oops_do(&cl); 1005 } 1006 1007 void ShenandoahVerifier::verify_roots_no_forwarded_except(ShenandoahRootVerifier::RootTypes types) { 1008 ShenandoahRootVerifier verifier; 1009 verifier.excludes(types); 1010 ShenandoahVerifyNoForwared cl; 1011 verifier.oops_do(&cl); 1012 }