1 /* 2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 // Deliberately avoids precompiled headers, to break name clashes and accidental dependencies 25 26 #include "gc/shenandoah/brooksPointer.hpp" 27 #include "gc/shenandoah/shenandoahAsserts.hpp" 28 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp" 29 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 30 #include "gc/shenandoah/shenandoahHeap.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 #include "gc/shenandoah/shenandoahRootProcessor.hpp" 33 #include "gc/shenandoah/shenandoahTaskqueue.hpp" 34 #include "gc/shenandoah/shenandoahVerifier.hpp" 35 #include "memory/allocation.hpp" 36 37 class ShenandoahVerifyOopClosure : public ExtendedOopClosure { 38 private: 39 const char* _phase; 40 ShenandoahVerifier::VerifyOptions _options; 41 ShenandoahVerifierStack* _stack; 42 ShenandoahHeap* _heap; 43 MarkBitMap* _map; 44 ShenandoahLivenessData* _ld; 45 void* _interior_loc; 46 oop _loc; 47 48 public: 49 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld, 50 const char* phase, ShenandoahVerifier::VerifyOptions options) : 51 _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL), 52 _phase(phase), _options(options) {}; 53 54 private: 55 void verify(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) { 56 if (!test) { 57 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__); 58 } 59 } 60 61 template <class T> 62 void do_oop_work(T* p) { 63 T o = RawAccess<>::oop_load(p); 64 if (!CompressedOops::is_null(o)) { 65 oop obj = CompressedOops::decode_not_null(o); 66 67 // Single threaded verification can use faster non-atomic stack and bitmap 68 // methods. 69 // 70 // For performance reasons, only fully verify non-marked field values. 71 // We are here when the host object for *p is already marked. If field value 72 // is marked already, then we have to verify the matrix connection between 73 // host object and field value, regardless. 74 75 HeapWord* addr = (HeapWord*) obj; 76 if (_map->parMark(addr)) { 77 verify_oop_at(p, obj); 78 _stack->push(ShenandoahVerifierTask(obj)); 79 } else { 80 verify_matrix(p, obj); 81 } 82 } 83 } 84 85 void verify_oop(oop obj) { 86 // Perform consistency checks with gradually decreasing safety level. This guarantees 87 // that failure report would not try to touch something that was not yet verified to be 88 // safe to process. 89 90 verify(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj), 91 "oop must be in heap"); 92 verify(ShenandoahAsserts::_safe_unknown, obj, check_obj_alignment(obj), 93 "oop must be aligned"); 94 95 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); 96 Klass* obj_klass = obj->klass_or_null(); 97 98 // Verify that obj is not in dead space: 99 { 100 // Do this before touching obj->size() 101 verify(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL, 102 "Object klass pointer should not be NULL"); 103 verify(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass), 104 "Object klass pointer must go to metaspace"); 105 106 HeapWord *obj_addr = (HeapWord *) obj; 107 verify(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(), 108 "Object start should be within the region"); 109 110 if (!obj_reg->is_humongous()) { 111 verify(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), 112 "Object end should be within the region"); 113 } else { 114 size_t humongous_start = obj_reg->region_number(); 115 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift()); 116 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) { 117 verify(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(), 118 "Humongous object is in continuation that fits it"); 119 } 120 } 121 122 // ------------ obj is safe at this point -------------- 123 124 verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(), 125 "Object should be in active region"); 126 127 switch (_options._verify_liveness) { 128 case ShenandoahVerifier::_verify_liveness_disable: 129 // skip 130 break; 131 case ShenandoahVerifier::_verify_liveness_complete: 132 Atomic::add(obj->size() + BrooksPointer::word_size(), &_ld[obj_reg->region_number()]); 133 // fallthrough for fast failure for un-live regions: 134 case ShenandoahVerifier::_verify_liveness_conservative: 135 verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(), 136 "Object must belong to region with live data"); 137 break; 138 default: 139 assert(false, "Unhandled liveness verification"); 140 } 141 } 142 143 oop fwd = (oop) BrooksPointer::get_raw_unchecked(obj); 144 145 ShenandoahHeapRegion* fwd_reg = NULL; 146 147 if (!oopDesc::unsafe_equals(obj, fwd)) { 148 verify(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd), 149 "Forwardee must be in heap"); 150 verify(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd), 151 "Forwardee is set"); 152 verify(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd), 153 "Forwardee must be aligned"); 154 155 // Do this before touching fwd->size() 156 Klass* fwd_klass = fwd->klass_or_null(); 157 verify(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL, 158 "Forwardee klass pointer should not be NULL"); 159 verify(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass), 160 "Forwardee klass pointer must go to metaspace"); 161 verify(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass, 162 "Forwardee klass pointer must go to metaspace"); 163 164 fwd_reg = _heap->heap_region_containing(fwd); 165 166 // Verify that forwardee is not in the dead space: 167 verify(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(), 168 "Should have no humongous forwardees"); 169 170 HeapWord *fwd_addr = (HeapWord *) fwd; 171 verify(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), 172 "Forwardee start should be within the region"); 173 verify(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), 174 "Forwardee end should be within the region"); 175 176 oop fwd2 = (oop) BrooksPointer::get_raw_unchecked(fwd); 177 verify(ShenandoahAsserts::_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2), 178 "Double forwarding"); 179 } else { 180 fwd_reg = obj_reg; 181 } 182 183 // ------------ obj and fwd are safe at this point -------------- 184 185 switch (_options._verify_marked) { 186 case ShenandoahVerifier::_verify_marked_disable: 187 // skip 188 break; 189 case ShenandoahVerifier::_verify_marked_next: 190 verify(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_next(obj), 191 "Must be marked in next bitmap"); 192 break; 193 case ShenandoahVerifier::_verify_marked_complete: 194 verify(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_complete(obj), 195 "Must be marked in complete bitmap"); 196 break; 197 default: 198 assert(false, "Unhandled mark verification"); 199 } 200 201 switch (_options._verify_forwarded) { 202 case ShenandoahVerifier::_verify_forwarded_disable: 203 // skip 204 break; 205 case ShenandoahVerifier::_verify_forwarded_none: { 206 verify(ShenandoahAsserts::_safe_all, obj, oopDesc::unsafe_equals(obj, fwd), 207 "Should not be forwarded"); 208 break; 209 } 210 case ShenandoahVerifier::_verify_forwarded_allow: { 211 if (!oopDesc::unsafe_equals(obj, fwd)) { 212 verify(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg, 213 "Forwardee should be in another region"); 214 } 215 break; 216 } 217 default: 218 assert(false, "Unhandled forwarding verification"); 219 } 220 221 switch (_options._verify_cset) { 222 case ShenandoahVerifier::_verify_cset_disable: 223 // skip 224 break; 225 case ShenandoahVerifier::_verify_cset_none: 226 verify(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj), 227 "Should not have references to collection set"); 228 break; 229 case ShenandoahVerifier::_verify_cset_forwarded: 230 if (_heap->in_collection_set(obj)) { 231 verify(ShenandoahAsserts::_safe_all, obj, !oopDesc::unsafe_equals(obj, fwd), 232 "Object in collection set, should have forwardee"); 233 } 234 break; 235 default: 236 assert(false, "Unhandled cset verification"); 237 } 238 239 verify_matrix(_interior_loc, obj); 240 } 241 242 void verify_matrix(void* interior, oop obj) { 243 if (!UseShenandoahMatrix || !_heap->is_in(interior)) return; 244 switch (_options._verify_matrix) { 245 case ShenandoahVerifier::_verify_matrix_conservative: { 246 size_t from_idx = _heap->heap_region_index_containing(interior); 247 size_t to_idx = _heap->heap_region_index_containing(obj); 248 _interior_loc = interior; 249 verify(ShenandoahAsserts::_safe_all, obj, _heap->connection_matrix()->is_connected(from_idx, to_idx), 250 "Must be connected"); 251 _interior_loc = NULL; 252 break; 253 } 254 case ShenandoahVerifier::_verify_matrix_disable: 255 break; 256 default: 257 assert(false, "Unhandled matrix verification"); 258 } 259 } 260 261 public: 262 263 /** 264 * Verify object with known interior reference. 265 * @param p interior reference where the object is referenced from; can be off-heap 266 * @param obj verified object 267 */ 268 template <class T> 269 void verify_oop_at(T* p, oop obj) { 270 _interior_loc = p; 271 verify_oop(obj); 272 _interior_loc = NULL; 273 } 274 275 /** 276 * Verify object without known interior reference. 277 * Useful when picking up the object at known offset in heap, 278 * but without knowing what objects reference it. 279 * @param obj verified object 280 */ 281 void verify_oop_standalone(oop obj) { 282 _interior_loc = NULL; 283 verify_oop(obj); 284 _interior_loc = NULL; 285 } 286 287 /** 288 * Verify oop fields from this object. 289 * @param obj host object for verified fields 290 */ 291 void verify_oops_from(oop obj) { 292 _loc = obj; 293 obj->oop_iterate(this); 294 _loc = NULL; 295 } 296 297 void do_oop(oop* p) { do_oop_work(p); } 298 void do_oop(narrowOop* p) { do_oop_work(p); } 299 }; 300 301 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { 302 private: 303 size_t _used, _committed, _garbage; 304 public: 305 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {}; 306 307 bool heap_region_do(ShenandoahHeapRegion* r) { 308 _used += r->used(); 309 _garbage += r->garbage(); 310 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; 311 return false; 312 } 313 314 size_t used() { return _used; } 315 size_t committed() { return _committed; } 316 size_t garbage() { return _garbage; } 317 }; 318 319 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { 320 private: 321 ShenandoahHeap* _heap; 322 const char* _phase; 323 ShenandoahVerifier::VerifyRegions _regions; 324 public: 325 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) : 326 _heap(ShenandoahHeap::heap()), _regions(regions), _phase(phase) {}; 327 328 void print_failure(ShenandoahHeapRegion* r, const char* label) { 329 ResourceMark rm; 330 331 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label); 332 333 stringStream ss; 334 r->print_on(&ss); 335 msg.append("%s", ss.as_string()); 336 337 report_vm_error(__FILE__, __LINE__, msg.buffer()); 338 } 339 340 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) { 341 if (!test) { 342 print_failure(r, msg); 343 } 344 } 345 346 bool heap_region_do(ShenandoahHeapRegion* r) { 347 switch (_regions) { 348 case ShenandoahVerifier::_verify_regions_disable: 349 break; 350 case ShenandoahVerifier::_verify_regions_notrash: 351 verify(r, !r->is_trash(), 352 "Should not have trash regions"); 353 break; 354 case ShenandoahVerifier::_verify_regions_nocset: 355 verify(r, !r->is_cset(), 356 "Should not have cset regions"); 357 break; 358 case ShenandoahVerifier::_verify_regions_notrash_nocset: 359 verify(r, !r->is_trash(), 360 "Should not have trash regions"); 361 verify(r, !r->is_cset(), 362 "Should not have cset regions"); 363 break; 364 default: 365 ShouldNotReachHere(); 366 } 367 368 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(), 369 "Capacity should match region size"); 370 371 verify(r, r->bottom() <= _heap->complete_top_at_mark_start(r->bottom()), 372 "Region top should not be less than bottom"); 373 374 verify(r, _heap->complete_top_at_mark_start(r->bottom()) <= r->top(), 375 "Complete TAMS should not be larger than top"); 376 377 verify(r, r->get_live_data_bytes() <= r->capacity(), 378 "Live data cannot be larger than capacity"); 379 380 verify(r, r->garbage() <= r->capacity(), 381 "Garbage cannot be larger than capacity"); 382 383 verify(r, r->used() <= r->capacity(), 384 "Used cannot be larger than capacity"); 385 386 verify(r, r->get_shared_allocs() <= r->capacity(), 387 "Shared alloc count should not be larger than capacity"); 388 389 verify(r, r->get_tlab_allocs() <= r->capacity(), 390 "TLAB alloc count should not be larger than capacity"); 391 392 verify(r, r->get_gclab_allocs() <= r->capacity(), 393 "GCLAB alloc count should not be larger than capacity"); 394 395 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(), 396 "Accurate accounting: shared + TLAB + GCLAB = used"); 397 398 verify(r, !r->is_empty() || !r->has_live(), 399 "Empty regions should not have live data"); 400 401 verify(r, r->is_cset() == r->in_collection_set(), 402 "Transitional: region flags and collection set agree"); 403 404 verify(r, r->is_empty() || r->seqnum_first_alloc() != 0, 405 "Non-empty regions should have first seqnum set"); 406 407 verify(r, r->is_empty() || (r->seqnum_first_alloc_mutator() != 0 || r->seqnum_first_alloc_gc() != 0), 408 "Non-empty regions should have first seqnum set to either GC or mutator"); 409 410 verify(r, r->is_empty() || r->seqnum_last_alloc() != 0, 411 "Non-empty regions should have last seqnum set"); 412 413 verify(r, r->is_empty() || (r->seqnum_last_alloc_mutator() != 0 || r->seqnum_last_alloc_gc() != 0), 414 "Non-empty regions should have last seqnum set to either GC or mutator"); 415 416 verify(r, r->seqnum_first_alloc() <= r->seqnum_last_alloc(), 417 "First seqnum should not be greater than last timestamp"); 418 419 verify(r, r->seqnum_first_alloc_mutator() <= r->seqnum_last_alloc_mutator(), 420 "First mutator seqnum should not be greater than last seqnum"); 421 422 verify(r, r->seqnum_first_alloc_gc() <= r->seqnum_last_alloc_gc(), 423 "First GC seqnum should not be greater than last seqnum"); 424 425 return false; 426 } 427 }; 428 429 class ShenandoahVerifierReachableTask : public AbstractGangTask { 430 private: 431 const char* _label; 432 ShenandoahRootProcessor* _rp; 433 ShenandoahVerifier::VerifyOptions _options; 434 ShenandoahHeap* _heap; 435 ShenandoahLivenessData* _ld; 436 MarkBitMap* _bitmap; 437 volatile size_t _processed; 438 439 public: 440 ShenandoahVerifierReachableTask(MarkBitMap* bitmap, 441 ShenandoahLivenessData* ld, 442 ShenandoahRootProcessor* rp, 443 const char* label, 444 ShenandoahVerifier::VerifyOptions options) : 445 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"), 446 _heap(ShenandoahHeap::heap()), _rp(rp), _ld(ld), _bitmap(bitmap), _processed(0), 447 _label(label), _options(options) {}; 448 449 size_t processed() { 450 return _processed; 451 } 452 453 virtual void work(uint worker_id) { 454 ResourceMark rm; 455 ShenandoahVerifierStack stack; 456 457 // On level 2, we need to only check the roots once. 458 // On level 3, we want to check the roots, and seed the local stack. 459 // It is a lesser evil to accept multiple root scans at level 3, because 460 // extended parallelism would buy us out. 461 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0)) 462 || (ShenandoahVerifyLevel >= 3)) { 463 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 464 ShenandoahMessageBuffer("%s, Roots", _label), 465 _options); 466 _rp->process_all_roots_slow(&cl); 467 } 468 469 size_t processed = 0; 470 471 if (ShenandoahVerifyLevel >= 3) { 472 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 473 ShenandoahMessageBuffer("%s, Reachable", _label), 474 _options); 475 while (!stack.is_empty()) { 476 processed++; 477 ShenandoahVerifierTask task = stack.pop(); 478 cl.verify_oops_from(task.obj()); 479 } 480 } 481 482 Atomic::add(processed, &_processed); 483 } 484 }; 485 486 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { 487 private: 488 const char* _label; 489 ShenandoahVerifier::VerifyOptions _options; 490 ShenandoahHeap *_heap; 491 MarkBitMap* _bitmap; 492 ShenandoahLivenessData* _ld; 493 volatile size_t _claimed; 494 volatile size_t _processed; 495 496 public: 497 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap, 498 ShenandoahLivenessData* ld, 499 const char* label, 500 ShenandoahVerifier::VerifyOptions options) : 501 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"), 502 _heap(ShenandoahHeap::heap()), _ld(ld), _bitmap(bitmap), _claimed(0), _processed(0), 503 _label(label), _options(options) {}; 504 505 size_t processed() { 506 return _processed; 507 } 508 509 virtual void work(uint worker_id) { 510 ShenandoahVerifierStack stack; 511 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 512 ShenandoahMessageBuffer("%s, Marked", _label), 513 _options); 514 515 while (true) { 516 size_t v = Atomic::add(1u, &_claimed) - 1; 517 if (v < _heap->num_regions()) { 518 ShenandoahHeapRegion* r = _heap->get_region(v); 519 if (!r->is_humongous() && !r->is_trash()) { 520 work_regular(r, stack, cl); 521 } else if (r->is_humongous_start()) { 522 work_humongous(r, stack, cl); 523 } 524 } else { 525 break; 526 } 527 } 528 } 529 530 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { 531 size_t processed = 0; 532 HeapWord* obj = r->bottom() + BrooksPointer::word_size(); 533 if (_heap->is_marked_complete((oop)obj)) { 534 verify_and_follow(obj, stack, cl, &processed); 535 } 536 Atomic::add(processed, &_processed); 537 } 538 539 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { 540 size_t processed = 0; 541 MarkBitMap* mark_bit_map = _heap->complete_mark_bit_map(); 542 HeapWord* tams = _heap->complete_top_at_mark_start(r->bottom()); 543 544 // Bitmaps, before TAMS 545 if (tams > r->bottom()) { 546 HeapWord* start = r->bottom() + BrooksPointer::word_size(); 547 HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams); 548 549 while (addr < tams) { 550 verify_and_follow(addr, stack, cl, &processed); 551 addr += BrooksPointer::word_size(); 552 if (addr < tams) { 553 addr = mark_bit_map->getNextMarkedWordAddress(addr, tams); 554 } 555 } 556 } 557 558 // Size-based, after TAMS 559 { 560 HeapWord* limit = r->top(); 561 HeapWord* addr = tams + BrooksPointer::word_size(); 562 563 while (addr < limit) { 564 verify_and_follow(addr, stack, cl, &processed); 565 addr += oop(addr)->size() + BrooksPointer::word_size(); 566 } 567 } 568 569 Atomic::add(processed, &_processed); 570 } 571 572 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) { 573 if (!_bitmap->parMark(addr)) return; 574 575 // Verify the object itself: 576 oop obj = oop(addr); 577 cl.verify_oop_standalone(obj); 578 579 // Verify everything reachable from that object too, hopefully realizing 580 // everything was already marked, and never touching further: 581 cl.verify_oops_from(obj); 582 (*processed)++; 583 584 while (!stack.is_empty()) { 585 ShenandoahVerifierTask task = stack.pop(); 586 cl.verify_oops_from(task.obj()); 587 (*processed)++; 588 } 589 } 590 }; 591 592 void ShenandoahVerifier::verify_at_safepoint(const char *label, 593 VerifyForwarded forwarded, VerifyMarked marked, 594 VerifyMatrix matrix, VerifyCollectionSet cset, 595 VerifyLiveness liveness, VerifyRegions regions) { 596 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens"); 597 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize"); 598 599 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check 600 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/); 601 602 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel); 603 604 // Heap size checks 605 { 606 ShenandoahHeapLocker lock(_heap->lock()); 607 608 ShenandoahCalculateRegionStatsClosure cl; 609 _heap->heap_region_iterate(&cl); 610 size_t heap_used = _heap->used(); 611 guarantee(cl.used() == heap_used, 612 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "K, regions-used = " SIZE_FORMAT "K", 613 label, heap_used/K, cl.used()/K); 614 615 size_t heap_committed = _heap->committed(); 616 guarantee(cl.committed() == heap_committed, 617 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "K, regions-committed = " SIZE_FORMAT "K", 618 label, heap_committed/K, cl.committed()/K); 619 } 620 621 // Internal heap region checks 622 if (ShenandoahVerifyLevel >= 1) { 623 ShenandoahVerifyHeapRegionClosure cl(label, regions); 624 _heap->heap_region_iterate(&cl, 625 /* skip_cset = */ false, 626 /* skip_humongous_cont = */ false); 627 } 628 629 OrderAccess::fence(); 630 _heap->make_tlabs_parsable(false); 631 632 // Allocate temporary bitmap for storing marking wavefront: 633 MemRegion mr = MemRegion(_verification_bit_map->startWord(), _verification_bit_map->endWord()); 634 _verification_bit_map->clear_range_large(mr); 635 636 // Allocate temporary array for storing liveness data 637 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC); 638 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0); 639 640 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, matrix, cset, liveness, regions); 641 642 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap. 643 // This verifies what application can see, since it only cares about reachable objects. 644 size_t count_reachable = 0; 645 if (ShenandoahVerifyLevel >= 2) { 646 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), 647 ShenandoahPhaseTimings::_num_phases); // no need for stats 648 649 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options); 650 _heap->workers()->run_task(&task); 651 count_reachable = task.processed(); 652 } 653 654 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector, 655 // not the application, can see during the region scans. There is no reason to process the objects 656 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS: 657 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics 658 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect) 659 // version 660 661 size_t count_marked = 0; 662 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) { 663 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options); 664 _heap->workers()->run_task(&task); 665 count_marked = task.processed(); 666 } else { 667 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_next || marked == _verify_marked_disable, "Should be"); 668 } 669 670 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes 671 // marked objects. 672 673 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) { 674 for (size_t i = 0; i < _heap->num_regions(); i++) { 675 ShenandoahHeapRegion* r = _heap->get_region(i); 676 677 juint verf_live = 0; 678 if (r->is_humongous()) { 679 // For humongous objects, test if start region is marked live, and if so, 680 // all humongous regions in that chain have live data equal to their "used". 681 juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]); 682 if (start_live > 0) { 683 verf_live = (juint)(r->used() / HeapWordSize); 684 } 685 } else { 686 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]); 687 } 688 689 size_t reg_live = r->get_live_data_words(); 690 if (reg_live != verf_live) { 691 ResourceMark rm; 692 stringStream ss; 693 r->print_on(&ss); 694 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s", 695 label, reg_live, verf_live, ss.as_string()); 696 } 697 } 698 } 699 700 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)", 701 label, ShenandoahVerifyLevel, count_reachable, count_marked); 702 703 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld); 704 } 705 706 void ShenandoahVerifier::verify_generic(VerifyOption vo) { 707 verify_at_safepoint( 708 "Generic Verification", 709 _verify_forwarded_allow, // conservatively allow forwarded 710 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 711 _verify_matrix_disable, // matrix can be inconsistent here 712 _verify_cset_disable, // cset may be inconsistent 713 _verify_liveness_disable, // no reliable liveness data 714 _verify_regions_disable // no reliable region data 715 ); 716 } 717 718 void ShenandoahVerifier::verify_before_concmark() { 719 if (_heap->has_forwarded_objects()) { 720 verify_at_safepoint( 721 "Before Mark", 722 _verify_forwarded_allow, // may have forwarded references 723 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 724 _verify_matrix_disable, // matrix is foobared 725 _verify_cset_forwarded, // allow forwarded references to cset 726 _verify_liveness_disable, // no reliable liveness data 727 _verify_regions_notrash // no trash regions 728 ); 729 } else { 730 verify_at_safepoint( 731 "Before Mark", 732 _verify_forwarded_none, // UR should have fixed up 733 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 734 _verify_matrix_conservative, // UR should have fixed matrix 735 _verify_cset_none, // UR should have fixed this 736 _verify_liveness_disable, // no reliable liveness data 737 _verify_regions_notrash // no trash regions 738 ); 739 } 740 } 741 742 void ShenandoahVerifier::verify_after_concmark() { 743 verify_at_safepoint( 744 "After Mark", 745 _verify_forwarded_none, // no forwarded references 746 _verify_marked_complete, // bitmaps as precise as we can get 747 _verify_matrix_disable, // matrix might be foobared 748 _verify_cset_none, // no references to cset anymore 749 _verify_liveness_complete, // liveness data must be complete here 750 _verify_regions_disable // trash regions not yet recycled 751 ); 752 } 753 754 void ShenandoahVerifier::verify_before_evacuation() { 755 // Evacuation is always preceded by mark, but we want to have a sanity check after 756 // selecting the collection set, and (immediate) regions recycling 757 verify_at_safepoint( 758 "Before Evacuation", 759 _verify_forwarded_none, // no forwarded references 760 _verify_marked_complete, // walk over marked objects too 761 _verify_matrix_disable, // skip, verified after mark 762 _verify_cset_disable, // skip, verified after mark 763 _verify_liveness_disable, // skip, verified after mark 764 _verify_regions_disable // trash regions not yet recycled 765 ); 766 } 767 768 void ShenandoahVerifier::verify_after_evacuation() { 769 verify_at_safepoint( 770 "After Evacuation", 771 _verify_forwarded_allow, // objects are still forwarded 772 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 773 _verify_matrix_disable, // matrix is inconsistent here 774 _verify_cset_forwarded, // all cset refs are fully forwarded 775 _verify_liveness_disable, // no reliable liveness data anymore 776 _verify_regions_notrash // trash regions have been recycled already 777 ); 778 } 779 780 void ShenandoahVerifier::verify_before_updaterefs() { 781 verify_at_safepoint( 782 "Before Updating References", 783 _verify_forwarded_allow, // forwarded references allowed 784 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 785 _verify_matrix_disable, // matrix is inconsistent here 786 _verify_cset_forwarded, // all cset refs are fully forwarded 787 _verify_liveness_disable, // no reliable liveness data anymore 788 _verify_regions_notrash // trash regions have been recycled already 789 ); 790 } 791 792 void ShenandoahVerifier::verify_after_updaterefs() { 793 verify_at_safepoint( 794 "After Updating References", 795 _verify_forwarded_none, // no forwarded references 796 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 797 _verify_matrix_conservative, // matrix is conservatively consistent 798 _verify_cset_none, // no cset references, all updated 799 _verify_liveness_disable, // no reliable liveness data anymore 800 _verify_regions_nocset // no cset regions, trash regions have appeared 801 ); 802 } 803 804 void ShenandoahVerifier::verify_after_degenerated() { 805 verify_at_safepoint( 806 "After Degenerated GC", 807 _verify_forwarded_none, // all objects are non-forwarded 808 _verify_marked_complete, // all objects are marked in complete bitmap 809 _verify_matrix_conservative, // matrix is conservatively consistent 810 _verify_cset_none, // no cset references 811 _verify_liveness_disable, // no reliable liveness data anymore 812 _verify_regions_notrash_nocset // no trash, no cset 813 ); 814 } 815 816 void ShenandoahVerifier::verify_before_partial() { 817 verify_at_safepoint( 818 "Before Partial", 819 _verify_forwarded_none, // cannot have forwarded objects 820 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 821 _verify_matrix_conservative, // matrix is conservatively consistent 822 _verify_cset_none, // no cset references before partial 823 _verify_liveness_disable, // no reliable liveness data anymore 824 _verify_regions_notrash_nocset // no trash and no cset regions 825 ); 826 } 827 828 void ShenandoahVerifier::verify_after_partial() { 829 verify_at_safepoint( 830 "After Partial", 831 _verify_forwarded_none, // cannot have forwarded objects 832 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 833 _verify_matrix_conservative, // matrix is conservatively consistent 834 _verify_cset_none, // no cset references left after partial 835 _verify_liveness_disable, // no reliable liveness data anymore 836 _verify_regions_nocset // no cset regions, trash regions allowed 837 ); 838 } 839 840 void ShenandoahVerifier::verify_before_traversal() { 841 verify_at_safepoint( 842 "Before Traversal", 843 _verify_forwarded_none, // cannot have forwarded objects 844 _verify_marked_disable, // bitmaps are not relevant before traversal 845 _verify_matrix_disable, // matrix is not used in traversal 846 _verify_cset_none, // no cset references before traversal 847 _verify_liveness_disable, // no reliable liveness data anymore 848 _verify_regions_notrash_nocset // no trash and no cset regions 849 ); 850 } 851 852 void ShenandoahVerifier::verify_after_traversal() { 853 verify_at_safepoint( 854 "After Traversal", 855 _verify_forwarded_none, // cannot have forwarded objects 856 _verify_marked_next, // marking should be complete in next bitmap 857 _verify_matrix_disable, // matrix is conservatively consistent 858 _verify_cset_none, // no cset references left after traversal 859 _verify_liveness_disable, // liveness data is not collected for new allocations 860 _verify_regions_nocset // no cset regions, trash regions allowed 861 ); 862 } 863 864 void ShenandoahVerifier::verify_before_fullgc() { 865 verify_at_safepoint( 866 "Before Full GC", 867 _verify_forwarded_allow, // can have forwarded objects 868 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 869 _verify_matrix_disable, // matrix might be foobared 870 _verify_cset_disable, // cset might be foobared 871 _verify_liveness_disable, // no reliable liveness data anymore 872 _verify_regions_disable // no reliable region data here 873 ); 874 } 875 876 void ShenandoahVerifier::verify_after_fullgc() { 877 verify_at_safepoint( 878 "After Full GC", 879 _verify_forwarded_none, // all objects are non-forwarded 880 _verify_marked_complete, // all objects are marked in complete bitmap 881 _verify_matrix_conservative, // matrix is conservatively consistent 882 _verify_cset_none, // no cset references 883 _verify_liveness_disable, // no reliable liveness data anymore 884 _verify_regions_notrash_nocset // no trash, no cset 885 ); 886 } 887