1 /* 2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 #include "precompiled.hpp" 24 25 #include "gc/shenandoah/brooksPointer.hpp" 26 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp" 27 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 28 #include "gc/shenandoah/shenandoahHeap.hpp" 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 30 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 31 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" 32 #include "gc/shenandoah/shenandoahRootProcessor.hpp" 33 #include "gc/shenandoah/shenandoahTaskqueue.hpp" 34 #include "gc/shenandoah/shenandoahVerifier.hpp" 35 #include "gc/shenandoah/shenandoahWorkGroup.hpp" 36 #include "memory/allocation.hpp" 37 38 class ShenandoahVerifyOopClosure : public ExtendedOopClosure { 39 private: 40 const char* _phase; 41 ShenandoahVerifier::VerifyOptions _options; 42 ShenandoahVerifierStack* _stack; 43 ShenandoahHeap* _heap; 44 MarkBitMap* _map; 45 ShenandoahLivenessData* _ld; 46 void* _interior_loc; 47 oop _loc; 48 49 public: 50 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld, 51 const char* phase, ShenandoahVerifier::VerifyOptions options) : 52 _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL), 53 _phase(phase), _options(options) {}; 54 55 private: 56 void print_obj(ShenandoahMessageBuffer& msg, oop obj) { 57 ShenandoahHeapRegion *r = _heap->heap_region_containing(obj); 58 stringStream ss; 59 r->print_on(&ss); 60 61 msg.append(" " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name()); 62 msg.append(" %3s allocated after complete mark start\n", _heap->allocated_after_complete_mark_start((HeapWord *) obj) ? "" : "not"); 63 msg.append(" %3s allocated after next mark start\n", _heap->allocated_after_next_mark_start((HeapWord *) obj) ? "" : "not"); 64 msg.append(" %3s marked complete\n", _heap->is_marked_complete(obj) ? "" : "not"); 65 msg.append(" %3s marked next\n", _heap->is_marked_next(obj) ? "" : "not"); 66 msg.append(" %3s in collection set\n", _heap->in_collection_set(obj) ? "" : "not"); 67 msg.append(" region: %s", ss.as_string()); 68 } 69 70 void print_non_obj(ShenandoahMessageBuffer& msg, void* loc) { 71 msg.append(" outside of Java heap\n"); 72 stringStream ss; 73 os::print_location(&ss, (intptr_t) loc, false); 74 msg.append(" %s\n", ss.as_string()); 75 } 76 77 void print_obj_safe(ShenandoahMessageBuffer& msg, void* loc) { 78 msg.append(" " PTR_FORMAT " - safe print, no details\n", p2i(loc)); 79 if (_heap->is_in(loc)) { 80 ShenandoahHeapRegion* r = _heap->heap_region_containing(loc); 81 if (r != NULL) { 82 stringStream ss; 83 r->print_on(&ss); 84 msg.append(" region: %s", ss.as_string()); 85 } 86 } 87 } 88 89 enum SafeLevel { 90 _safe_unknown, 91 _safe_oop, 92 _safe_oop_fwd, 93 _safe_all, 94 }; 95 96 void print_failure(SafeLevel level, oop obj, const char* label) { 97 ResourceMark rm; 98 99 bool loc_in_heap = (_loc != NULL && _heap->is_in(_loc)); 100 bool interior_loc_in_heap = (_interior_loc != NULL && _heap->is_in(_interior_loc)); 101 102 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label); 103 104 msg.append("Referenced from:\n"); 105 if (_interior_loc != NULL) { 106 msg.append(" interior location: " PTR_FORMAT "\n", p2i(_interior_loc)); 107 if (loc_in_heap) { 108 print_obj(msg, _loc); 109 } else { 110 print_non_obj(msg, _interior_loc); 111 } 112 } else { 113 msg.append(" no location recorded, probably a plain heap scan\n"); 114 } 115 msg.append("\n"); 116 117 msg.append("Object:\n"); 118 if (level >= _safe_oop) { 119 print_obj(msg, obj); 120 } else { 121 print_obj_safe(msg, obj); 122 } 123 msg.append("\n"); 124 125 if (level >= _safe_oop) { 126 oop fwd = (oop) BrooksPointer::get_raw(obj); 127 if (!oopDesc::unsafe_equals(obj, fwd)) { 128 msg.append("Forwardee:\n"); 129 if (level >= _safe_oop_fwd) { 130 print_obj(msg, fwd); 131 } else { 132 print_obj_safe(msg, fwd); 133 } 134 msg.append("\n"); 135 } 136 } 137 138 if (level >= _safe_oop_fwd) { 139 oop fwd = (oop) BrooksPointer::get_raw(obj); 140 oop fwd2 = (oop) BrooksPointer::get_raw(fwd); 141 if (!oopDesc::unsafe_equals(fwd, fwd2)) { 142 msg.append("Second forwardee:\n"); 143 print_obj_safe(msg, fwd2); 144 msg.append("\n"); 145 } 146 } 147 148 if (loc_in_heap && UseShenandoahMatrix && (level == _safe_all)) { 149 msg.append("Matrix connections:\n"); 150 151 oop fwd_to = (oop) BrooksPointer::get_raw(obj); 152 oop fwd_from = (oop) BrooksPointer::get_raw(_loc); 153 154 size_t from_idx = _heap->heap_region_index_containing(_loc); 155 size_t to_idx = _heap->heap_region_index_containing(obj); 156 size_t fwd_from_idx = _heap->heap_region_index_containing(fwd_from); 157 size_t fwd_to_idx = _heap->heap_region_index_containing(fwd_to); 158 159 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix(); 160 msg.append(" %35s %3s connected\n", 161 "reference and object", 162 matrix->is_connected(from_idx, to_idx) ? "" : "not"); 163 msg.append(" %35s %3s connected\n", 164 "fwd(reference) and object", 165 matrix->is_connected(fwd_from_idx, to_idx) ? "" : "not"); 166 msg.append(" %35s %3s connected\n", 167 "reference and fwd(object)", 168 matrix->is_connected(from_idx, fwd_to_idx) ? "" : "not"); 169 msg.append(" %35s %3s connected\n", 170 "fwd(reference) and fwd(object)", 171 matrix->is_connected(fwd_from_idx, fwd_to_idx) ? "" : "not"); 172 173 if (interior_loc_in_heap) { 174 size_t from_interior_idx = _heap->heap_region_index_containing(_interior_loc); 175 msg.append(" %35s %3s connected\n", 176 "interior-reference and object", 177 matrix->is_connected(from_interior_idx, to_idx) ? "" : "not"); 178 msg.append(" %35s %3s connected\n", 179 "interior-reference and fwd(object)", 180 matrix->is_connected(from_interior_idx, fwd_to_idx) ? "" : "not"); 181 } 182 } 183 184 report_vm_error(__FILE__, __LINE__, msg.buffer()); 185 } 186 187 void verify(SafeLevel level, oop obj, bool test, const char* label) { 188 if (!test) { 189 print_failure(level, obj, label); 190 } 191 } 192 193 template <class T> 194 void do_oop_work(T* p) { 195 T o = oopDesc::load_heap_oop(p); 196 if (!oopDesc::is_null(o)) { 197 oop obj = oopDesc::decode_heap_oop_not_null(o); 198 199 // Single threaded verification can use faster non-atomic stack and bitmap 200 // methods. 201 // 202 // For performance reasons, only fully verify non-marked field values. 203 // We are here when the host object for *p is already marked. If field value 204 // is marked already, then we have to verify the matrix connection between 205 // host object and field value, regardless. 206 207 HeapWord* addr = (HeapWord*) obj; 208 if (_map->parMark(addr)) { 209 verify_oop_at(p, obj); 210 _stack->push(ShenandoahVerifierTask(obj)); 211 } else { 212 verify_matrix(p, obj); 213 } 214 } 215 } 216 217 void verify_oop(oop obj) { 218 // Perform consistency checks with gradually decreasing safety level. This guarantees 219 // that failure report would not try to touch something that was not yet verified to be 220 // safe to process. 221 222 verify(_safe_unknown, obj, _heap->is_in(obj), 223 "oop must be in heap"); 224 verify(_safe_unknown, obj, check_obj_alignment(obj), 225 "oop must be aligned"); 226 227 // Verify that obj is not in dead space: 228 { 229 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); 230 231 HeapWord *obj_addr = (HeapWord *) obj; 232 verify(_safe_unknown, obj, obj_addr < obj_reg->top(), 233 "Object start should be within the region"); 234 235 if (!obj_reg->is_humongous()) { 236 verify(_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), 237 "Object end should be within the region"); 238 } else { 239 size_t humongous_start = obj_reg->region_number(); 240 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift()); 241 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) { 242 verify(_safe_unknown, obj, _heap->regions()->get(idx)->is_humongous_continuation(), 243 "Humongous object is in continuation that fits it"); 244 } 245 } 246 247 verify(_safe_unknown, obj, Metaspace::contains(obj->klass()), 248 "klass pointer must go to metaspace"); 249 250 // ------------ obj is safe at this point -------------- 251 252 verify(_safe_oop, obj, obj_reg->is_active(), 253 "Object should be in active region"); 254 255 switch (_options._verify_liveness) { 256 case ShenandoahVerifier::_verify_liveness_disable: 257 // skip 258 break; 259 case ShenandoahVerifier::_verify_liveness_complete: 260 Atomic::add(obj->size() + BrooksPointer::word_size(), &_ld[obj_reg->region_number()]); 261 // fallthrough for fast failure for un-live regions: 262 case ShenandoahVerifier::_verify_liveness_conservative: 263 verify(_safe_oop, obj, obj_reg->has_live(), 264 "Object must belong to region with live data"); 265 break; 266 default: 267 assert(false, "Unhandled liveness verification"); 268 } 269 } 270 271 oop fwd = (oop) BrooksPointer::get_raw(obj); 272 if (!oopDesc::unsafe_equals(obj, fwd)) { 273 verify(_safe_oop, obj, _heap->is_in(fwd), 274 "Forwardee must be in heap"); 275 verify(_safe_oop, obj, !oopDesc::is_null(fwd), 276 "Forwardee is set"); 277 verify(_safe_oop, obj, check_obj_alignment(fwd), 278 "Forwardee must be aligned"); 279 280 // Verify that forwardee is not in the dead space: 281 if (!oopDesc::unsafe_equals(obj, fwd)) { 282 ShenandoahHeapRegion *fwd_reg = _heap->heap_region_containing(fwd); 283 verify(_safe_oop, obj, !fwd_reg->is_humongous(), 284 "Should have no humongous forwardees"); 285 286 HeapWord *fwd_addr = (HeapWord *) fwd; 287 verify(_safe_oop, obj, fwd_addr < fwd_reg->top(), 288 "Forwardee start should be within the region"); 289 verify(_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), 290 "Forwardee end should be within the region"); 291 } 292 293 verify(_safe_oop, obj, Metaspace::contains(fwd->klass()), 294 "Forwardee klass pointer must go to metaspace"); 295 verify(_safe_oop, obj, obj->klass() == fwd->klass(), 296 "Forwardee klass pointer must go to metaspace"); 297 298 oop fwd2 = (oop) BrooksPointer::get_raw(fwd); 299 verify(_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2), 300 "Double forwarding"); 301 } 302 303 // ------------ obj and fwd are safe at this point -------------- 304 305 switch (_options._verify_marked) { 306 case ShenandoahVerifier::_verify_marked_disable: 307 // skip 308 break; 309 case ShenandoahVerifier::_verify_marked_next: 310 verify(_safe_all, obj, _heap->is_marked_next(obj), 311 "Must be marked in next bitmap"); 312 break; 313 case ShenandoahVerifier::_verify_marked_complete: 314 verify(_safe_all, obj, _heap->is_marked_complete(obj), 315 "Must be marked in complete bitmap"); 316 break; 317 default: 318 assert(false, "Unhandled mark verification"); 319 } 320 321 switch (_options._verify_forwarded) { 322 case ShenandoahVerifier::_verify_forwarded_disable: 323 // skip 324 break; 325 case ShenandoahVerifier::_verify_forwarded_none: { 326 verify(_safe_all, obj, oopDesc::unsafe_equals(obj, fwd), 327 "Should not be forwarded"); 328 break; 329 } 330 case ShenandoahVerifier::_verify_forwarded_allow: { 331 if (!oopDesc::unsafe_equals(obj, fwd)) { 332 verify(_safe_all, obj, _heap->heap_region_containing(obj) != _heap->heap_region_containing(fwd), 333 "Forwardee should be in another region"); 334 } 335 break; 336 } 337 default: 338 assert(false, "Unhandled forwarding verification"); 339 } 340 341 switch (_options._verify_cset) { 342 case ShenandoahVerifier::_verify_cset_disable: 343 // skip 344 break; 345 case ShenandoahVerifier::_verify_cset_none: 346 verify(_safe_all, obj, !_heap->in_collection_set(obj), 347 "Should not have references to collection set"); 348 break; 349 case ShenandoahVerifier::_verify_cset_forwarded: 350 if (_heap->in_collection_set(obj)) { 351 verify(_safe_all, obj, !oopDesc::unsafe_equals(obj, fwd), 352 "Object in collection set, should have forwardee"); 353 } 354 break; 355 default: 356 assert(false, "Unhandled cset verification"); 357 } 358 359 verify_matrix(_interior_loc, obj); 360 } 361 362 void verify_matrix(void* interior, oop obj) { 363 if (!UseShenandoahMatrix || !_heap->is_in(interior)) return; 364 switch (_options._verify_matrix) { 365 case ShenandoahVerifier::_verify_matrix_conservative: { 366 size_t from_idx = _heap->heap_region_index_containing(interior); 367 size_t to_idx = _heap->heap_region_index_containing(obj); 368 _interior_loc = interior; 369 verify(_safe_all, obj, _heap->connection_matrix()->is_connected(from_idx, to_idx), 370 "Must be connected"); 371 _interior_loc = NULL; 372 break; 373 } 374 case ShenandoahVerifier::_verify_matrix_disable: 375 break; 376 default: 377 assert(false, "Unhandled matrix verification"); 378 } 379 } 380 381 public: 382 383 /** 384 * Verify object with known interior reference. 385 * @param p interior reference where the object is referenced from; can be off-heap 386 * @param obj verified object 387 */ 388 template <class T> 389 void verify_oop_at(T* p, oop obj) { 390 _interior_loc = p; 391 verify_oop(obj); 392 _interior_loc = NULL; 393 } 394 395 /** 396 * Verify object without known interior reference. 397 * Useful when picking up the object at known offset in heap, 398 * but without knowing what objects reference it. 399 * @param obj verified object 400 */ 401 void verify_oop_standalone(oop obj) { 402 _interior_loc = NULL; 403 verify_oop(obj); 404 _interior_loc = NULL; 405 } 406 407 /** 408 * Verify oop fields from this object. 409 * @param obj host object for verified fields 410 */ 411 void verify_oops_from(oop obj) { 412 _loc = obj; 413 obj->oop_iterate(this); 414 _loc = NULL; 415 } 416 417 void do_oop(oop* p) { do_oop_work(p); } 418 void do_oop(narrowOop* p) { do_oop_work(p); } 419 }; 420 421 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { 422 private: 423 size_t _used, _committed, _garbage; 424 public: 425 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {}; 426 427 bool heap_region_do(ShenandoahHeapRegion* r) { 428 _used += r->used(); 429 _garbage += r->garbage(); 430 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; 431 return false; 432 } 433 434 size_t used() { return _used; } 435 size_t committed() { return _committed; } 436 size_t garbage() { return _garbage; } 437 }; 438 439 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { 440 private: 441 ShenandoahHeap* _heap; 442 const char* _phase; 443 ShenandoahVerifier::VerifyRegions _regions; 444 public: 445 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) : 446 _heap(ShenandoahHeap::heap()), _regions(regions), _phase(phase) {}; 447 448 void print_failure(ShenandoahHeapRegion* r, const char* label) { 449 ResourceMark rm; 450 451 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label); 452 453 stringStream ss; 454 r->print_on(&ss); 455 msg.append("%s", ss.as_string()); 456 457 report_vm_error(__FILE__, __LINE__, msg.buffer()); 458 } 459 460 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) { 461 if (!test) { 462 print_failure(r, msg); 463 } 464 } 465 466 bool heap_region_do(ShenandoahHeapRegion* r) { 467 switch (_regions) { 468 case ShenandoahVerifier::_verify_regions_disable: 469 break; 470 case ShenandoahVerifier::_verify_regions_notrash: 471 verify(r, !r->is_trash(), 472 "Should not have trash regions"); 473 break; 474 case ShenandoahVerifier::_verify_regions_nocset: 475 verify(r, !r->is_cset(), 476 "Should not have cset regions"); 477 break; 478 case ShenandoahVerifier::_verify_regions_notrash_nocset: 479 verify(r, !r->is_trash(), 480 "Should not have trash regions"); 481 verify(r, !r->is_cset(), 482 "Should not have cset regions"); 483 break; 484 default: 485 ShouldNotReachHere(); 486 } 487 488 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(), 489 "Capacity should match region size"); 490 491 verify(r, r->bottom() <= _heap->complete_top_at_mark_start(r->bottom()), 492 "Region top should not be less than bottom"); 493 494 verify(r, _heap->complete_top_at_mark_start(r->bottom()) <= r->top(), 495 "Complete TAMS should not be larger than top"); 496 497 verify(r, r->get_live_data_bytes() <= r->capacity(), 498 "Live data cannot be larger than capacity"); 499 500 verify(r, r->garbage() <= r->capacity(), 501 "Garbage cannot be larger than capacity"); 502 503 verify(r, r->used() <= r->capacity(), 504 "Used cannot be larger than capacity"); 505 506 verify(r, r->get_shared_allocs() <= r->capacity(), 507 "Shared alloc count should not be larger than capacity"); 508 509 verify(r, r->get_tlab_allocs() <= r->capacity(), 510 "TLAB alloc count should not be larger than capacity"); 511 512 verify(r, r->get_gclab_allocs() <= r->capacity(), 513 "GCLAB alloc count should not be larger than capacity"); 514 515 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(), 516 "Accurate accounting: shared + TLAB + GCLAB = used"); 517 518 verify(r, !r->is_empty() || !r->has_live(), 519 "Empty regions should not have live data"); 520 521 verify(r, r->is_cset() == r->in_collection_set(), 522 "Transitional: region flags and collection set agree"); 523 524 verify(r, r->is_empty() || r->first_alloc_seq_num() != 0, 525 "Non-empty regions should have first timestamp set"); 526 527 verify(r, r->is_empty() || r->last_alloc_seq_num() != 0, 528 "Non-empty regions should have last timestamp set"); 529 530 verify(r, r->first_alloc_seq_num() <= r->last_alloc_seq_num(), 531 "First timestamp should not be greater than last timestamp"); 532 533 return false; 534 } 535 }; 536 537 class ShenandoahVerifierReachableTask : public AbstractGangTask { 538 private: 539 const char* _label; 540 ShenandoahRootProcessor* _rp; 541 ShenandoahVerifier::VerifyOptions _options; 542 ShenandoahHeap* _heap; 543 ShenandoahLivenessData* _ld; 544 MarkBitMap* _bitmap; 545 volatile size_t _processed; 546 547 public: 548 ShenandoahVerifierReachableTask(MarkBitMap* bitmap, 549 ShenandoahLivenessData* ld, 550 ShenandoahRootProcessor* rp, 551 const char* label, 552 ShenandoahVerifier::VerifyOptions options) : 553 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"), 554 _heap(ShenandoahHeap::heap()), _rp(rp), _ld(ld), _bitmap(bitmap), _processed(0), 555 _label(label), _options(options) {}; 556 557 size_t processed() { 558 return _processed; 559 } 560 561 virtual void work(uint worker_id) { 562 ResourceMark rm; 563 ShenandoahVerifierStack stack; 564 565 // On level 2, we need to only check the roots once. 566 // On level 3, we want to check the roots, and seed the local stack. 567 // It is a lesser evil to accept multiple root scans at level 3, because 568 // extended parallelism would buy us out. 569 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0)) 570 || (ShenandoahVerifyLevel >= 3)) { 571 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 572 ShenandoahMessageBuffer("%s, Roots", _label), 573 _options); 574 _rp->process_all_roots_slow(&cl); 575 } 576 577 size_t processed = 0; 578 579 if (ShenandoahVerifyLevel >= 3) { 580 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 581 ShenandoahMessageBuffer("%s, Reachable", _label), 582 _options); 583 while (!stack.is_empty()) { 584 processed++; 585 ShenandoahVerifierTask task = stack.pop(); 586 cl.verify_oops_from(task.obj()); 587 } 588 } 589 590 Atomic::add(processed, &_processed); 591 } 592 }; 593 594 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { 595 private: 596 const char* _label; 597 ShenandoahVerifier::VerifyOptions _options; 598 ShenandoahHeap *_heap; 599 ShenandoahHeapRegionSet* _regions; 600 MarkBitMap* _bitmap; 601 ShenandoahLivenessData* _ld; 602 volatile size_t _claimed; 603 volatile size_t _processed; 604 605 public: 606 ShenandoahVerifierMarkedRegionTask(ShenandoahHeapRegionSet* regions, MarkBitMap* bitmap, 607 ShenandoahLivenessData* ld, 608 const char* label, 609 ShenandoahVerifier::VerifyOptions options) : 610 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"), 611 _heap(ShenandoahHeap::heap()), _regions(regions), _ld(ld), _bitmap(bitmap), _claimed(0), _processed(0), 612 _label(label), _options(options) {}; 613 614 size_t processed() { 615 return _processed; 616 } 617 618 virtual void work(uint worker_id) { 619 ShenandoahVerifierStack stack; 620 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, 621 ShenandoahMessageBuffer("%s, Marked", _label), 622 _options); 623 624 while (true) { 625 size_t v = Atomic::add(1u, &_claimed) - 1; 626 if (v < _heap->num_regions()) { 627 ShenandoahHeapRegion* r = _regions->get(v); 628 if (!r->is_humongous() && !r->is_trash()) { 629 work_regular(r, stack, cl); 630 } else if (r->is_humongous_start()) { 631 work_humongous(r, stack, cl); 632 } 633 } else { 634 break; 635 } 636 } 637 } 638 639 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { 640 size_t processed = 0; 641 HeapWord* obj = r->bottom() + BrooksPointer::word_size(); 642 if (_heap->is_marked_complete((oop)obj)) { 643 verify_and_follow(obj, stack, cl, &processed); 644 } 645 Atomic::add(processed, &_processed); 646 } 647 648 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { 649 size_t processed = 0; 650 MarkBitMap* mark_bit_map = _heap->complete_mark_bit_map(); 651 HeapWord* tams = _heap->complete_top_at_mark_start(r->bottom()); 652 653 // Bitmaps, before TAMS 654 if (tams > r->bottom()) { 655 HeapWord* start = r->bottom() + BrooksPointer::word_size(); 656 HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams); 657 658 while (addr < tams) { 659 verify_and_follow(addr, stack, cl, &processed); 660 addr += BrooksPointer::word_size(); 661 if (addr < tams) { 662 addr = mark_bit_map->getNextMarkedWordAddress(addr, tams); 663 } 664 } 665 } 666 667 // Size-based, after TAMS 668 { 669 HeapWord* limit = r->top(); 670 HeapWord* addr = tams + BrooksPointer::word_size(); 671 672 while (addr < limit) { 673 verify_and_follow(addr, stack, cl, &processed); 674 addr += oop(addr)->size() + BrooksPointer::word_size(); 675 } 676 } 677 678 Atomic::add(processed, &_processed); 679 } 680 681 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) { 682 if (!_bitmap->parMark(addr)) return; 683 684 // Verify the object itself: 685 oop obj = oop(addr); 686 cl.verify_oop_standalone(obj); 687 688 // Verify everything reachable from that object too, hopefully realizing 689 // everything was already marked, and never touching further: 690 cl.verify_oops_from(obj); 691 (*processed)++; 692 693 while (!stack.is_empty()) { 694 ShenandoahVerifierTask task = stack.pop(); 695 cl.verify_oops_from(task.obj()); 696 (*processed)++; 697 } 698 } 699 }; 700 701 void ShenandoahVerifier::verify_at_safepoint(const char *label, 702 VerifyForwarded forwarded, VerifyMarked marked, 703 VerifyMatrix matrix, VerifyCollectionSet cset, 704 VerifyLiveness liveness, VerifyRegions regions) { 705 guarantee(SafepointSynchronize::is_at_safepoint(), "only when nothing else happens"); 706 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize"); 707 708 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check 709 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/); 710 711 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel); 712 713 // Heap size checks 714 { 715 ShenandoahHeapLocker lock(_heap->lock()); 716 717 ShenandoahCalculateRegionStatsClosure cl; 718 _heap->heap_region_iterate(&cl); 719 size_t heap_used = _heap->used(); 720 guarantee(cl.used() == heap_used, 721 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "K, regions-used = " SIZE_FORMAT "K", 722 label, heap_used/K, cl.used()/K); 723 724 size_t heap_committed = _heap->committed(); 725 guarantee(cl.committed() == heap_committed, 726 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "K, regions-committed = " SIZE_FORMAT "K", 727 label, heap_committed/K, cl.committed()/K); 728 } 729 730 // Internal heap region checks 731 if (ShenandoahVerifyLevel >= 1) { 732 ShenandoahVerifyHeapRegionClosure cl(label, regions); 733 _heap->heap_region_iterate(&cl, 734 /* skip_cset = */ false, 735 /* skip_humongous_cont = */ false); 736 } 737 738 OrderAccess::fence(); 739 _heap->ensure_parsability(false); 740 741 // Allocate temporary bitmap for storing marking wavefront: 742 MemRegion mr = MemRegion(_verification_bit_map->startWord(), _verification_bit_map->endWord()); 743 _verification_bit_map->clear_range_large(mr); 744 745 // Allocate temporary array for storing liveness data 746 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC); 747 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0); 748 749 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, matrix, cset, liveness, regions); 750 751 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap. 752 // This verifies what application can see, since it only cares about reachable objects. 753 size_t count_reachable = 0; 754 if (ShenandoahVerifyLevel >= 2) { 755 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), 756 ShenandoahPhaseTimings::_num_phases); // no need for stats 757 758 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options); 759 _heap->workers()->run_task(&task); 760 count_reachable = task.processed(); 761 } 762 763 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector, 764 // not the application, can see during the region scans. There is no reason to process the objects 765 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS: 766 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics 767 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect) 768 // version 769 770 size_t count_marked = 0; 771 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) { 772 ShenandoahVerifierMarkedRegionTask task(_heap->regions(), _verification_bit_map, ld, label, options); 773 _heap->workers()->run_task(&task); 774 count_marked = task.processed(); 775 } else { 776 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_next || marked == _verify_marked_disable, "Should be"); 777 } 778 779 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes 780 // marked objects. 781 782 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) { 783 ShenandoahHeapRegionSet* set = _heap->regions(); 784 for (size_t i = 0; i < _heap->num_regions(); i++) { 785 ShenandoahHeapRegion* r = set->get(i); 786 787 juint verf_live = 0; 788 if (r->is_humongous()) { 789 // For humongous objects, test if start region is marked live, and if so, 790 // all humongous regions in that chain have live data equal to their "used". 791 juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]); 792 if (start_live > 0) { 793 verf_live = (juint)(r->used() / HeapWordSize); 794 } 795 } else { 796 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]); 797 } 798 799 size_t reg_live = r->get_live_data_words(); 800 if (reg_live != verf_live) { 801 ResourceMark rm; 802 stringStream ss; 803 r->print_on(&ss); 804 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s", 805 label, reg_live, verf_live, ss.as_string()); 806 } 807 } 808 } 809 810 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)", 811 label, ShenandoahVerifyLevel, count_reachable, count_marked); 812 813 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld); 814 } 815 816 void ShenandoahVerifier::verify_generic(VerifyOption vo) { 817 verify_at_safepoint( 818 "Generic Verification", 819 _verify_forwarded_allow, // conservatively allow forwarded 820 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 821 _verify_matrix_disable, // matrix can be inconsistent here 822 _verify_cset_disable, // cset may be inconsistent 823 _verify_liveness_disable, // no reliable liveness data 824 _verify_regions_disable // no reliable region data 825 ); 826 } 827 828 void ShenandoahVerifier::verify_before_concmark() { 829 if (_heap->need_update_refs()) { 830 verify_at_safepoint( 831 "Before Mark", 832 _verify_forwarded_allow, // may have forwarded references 833 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 834 _verify_matrix_disable, // matrix is foobared 835 _verify_cset_forwarded, // allow forwarded references to cset 836 _verify_liveness_disable, // no reliable liveness data 837 _verify_regions_notrash // no trash regions 838 ); 839 } else { 840 verify_at_safepoint( 841 "Before Mark", 842 _verify_forwarded_none, // UR should have fixed up 843 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 844 _verify_matrix_conservative, // UR should have fixed matrix 845 _verify_cset_none, // UR should have fixed this 846 _verify_liveness_disable, // no reliable liveness data 847 _verify_regions_notrash // no trash regions 848 ); 849 } 850 } 851 852 void ShenandoahVerifier::verify_after_concmark() { 853 verify_at_safepoint( 854 "After Mark", 855 _verify_forwarded_none, // no forwarded references 856 _verify_marked_complete, // bitmaps as precise as we can get 857 _verify_matrix_disable, // matrix might be foobared 858 _verify_cset_none, // no references to cset anymore 859 _verify_liveness_complete, // liveness data must be complete here 860 _verify_regions_disable // trash regions not yet recycled 861 ); 862 } 863 864 void ShenandoahVerifier::verify_before_evacuation() { 865 // Evacuation is always preceded by mark, but we want to have a sanity check after 866 // selecting the collection set, and (immediate) regions recycling 867 verify_at_safepoint( 868 "Before Evacuation", 869 _verify_forwarded_none, // no forwarded references 870 _verify_marked_complete, // walk over marked objects too 871 _verify_matrix_disable, // skip, verified after mark 872 _verify_cset_disable, // skip, verified after mark 873 _verify_liveness_disable, // skip, verified after mark 874 _verify_regions_disable // trash regions not yet recycled 875 ); 876 } 877 878 void ShenandoahVerifier::verify_after_evacuation() { 879 verify_at_safepoint( 880 "After Evacuation", 881 _verify_forwarded_allow, // objects are still forwarded 882 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 883 _verify_matrix_disable, // matrix is inconsistent here 884 _verify_cset_forwarded, // all cset refs are fully forwarded 885 _verify_liveness_disable, // no reliable liveness data anymore 886 _verify_regions_notrash // trash regions have been recycled already 887 ); 888 } 889 890 void ShenandoahVerifier::verify_before_updaterefs() { 891 verify_at_safepoint( 892 "Before Updating References", 893 _verify_forwarded_allow, // forwarded references allowed 894 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 895 _verify_matrix_disable, // matrix is inconsistent here 896 _verify_cset_forwarded, // all cset refs are fully forwarded 897 _verify_liveness_disable, // no reliable liveness data anymore 898 _verify_regions_notrash // trash regions have been recycled already 899 ); 900 } 901 902 void ShenandoahVerifier::verify_after_updaterefs() { 903 verify_at_safepoint( 904 "After Updating References", 905 _verify_forwarded_none, // no forwarded references 906 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 907 _verify_matrix_conservative, // matrix is conservatively consistent 908 _verify_cset_none, // no cset references, all updated 909 _verify_liveness_disable, // no reliable liveness data anymore 910 _verify_regions_nocset // no cset regions, trash regions have appeared 911 ); 912 } 913 914 void ShenandoahVerifier::verify_before_partial() { 915 verify_at_safepoint( 916 "Before Partial GC", 917 _verify_forwarded_none, // cannot have forwarded objects 918 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 919 _verify_matrix_conservative, // matrix is conservatively consistent 920 _verify_cset_none, // no cset references before partial 921 _verify_liveness_disable, // no reliable liveness data anymore 922 _verify_regions_notrash_nocset // no trash and no cset regions 923 ); 924 } 925 926 void ShenandoahVerifier::verify_after_partial() { 927 verify_at_safepoint( 928 "After Partial GC", 929 _verify_forwarded_none, // cannot have forwarded objects 930 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well 931 _verify_matrix_conservative, // matrix is conservatively consistent 932 _verify_cset_none, // no cset references left after partial 933 _verify_liveness_disable, // no reliable liveness data anymore 934 _verify_regions_nocset // no cset regions, trash regions allowed 935 ); 936 } 937 938 void ShenandoahVerifier::verify_before_fullgc() { 939 verify_at_safepoint( 940 "Before Full GC", 941 _verify_forwarded_allow, // can have forwarded objects 942 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations 943 _verify_matrix_disable, // matrix might be foobared 944 _verify_cset_disable, // cset might be foobared 945 _verify_liveness_disable, // no reliable liveness data anymore 946 _verify_regions_disable // no reliable region data here 947 ); 948 } 949 950 void ShenandoahVerifier::verify_after_fullgc() { 951 verify_at_safepoint( 952 "After Full GC", 953 _verify_forwarded_none, // all objects are non-forwarded 954 _verify_marked_complete, // all objects are marked in complete bitmap 955 _verify_matrix_conservative, // matrix is conservatively consistent 956 _verify_cset_none, // no cset references 957 _verify_liveness_disable, // no reliable liveness data anymore 958 _verify_regions_notrash_nocset // no trash, no cset 959 ); 960 } 961 962 void ShenandoahVerifier::verify_oop_fwdptr(oop obj, oop fwd) { 963 guarantee(UseShenandoahGC, "must only be called when Shenandoah is used"); 964 965 ShenandoahHeap* heap = ShenandoahHeap::heap(); 966 967 guarantee(obj != NULL, "oop is not NULL"); 968 guarantee(heap->is_in(obj), "oop must point to a heap address: " PTR_FORMAT, p2i(obj)); 969 970 guarantee(fwd != NULL, "forwardee is not NULL"); 971 if (!heap->is_in(fwd)) { 972 ResourceMark rm; 973 ShenandoahHeapRegion* r = heap->heap_region_containing(obj); 974 stringStream obj_region; 975 r->print_on(&obj_region); 976 977 fatal("forwardee must point to a heap address: " PTR_FORMAT " -> " PTR_FORMAT "\n %s", 978 p2i(obj), p2i(fwd), obj_region.as_string()); 979 } 980 981 if (!oopDesc::unsafe_equals(fwd, obj) && 982 (heap->heap_region_containing(fwd) == 983 heap->heap_region_containing(obj))) { 984 ResourceMark rm; 985 ShenandoahHeapRegion* ro = heap->heap_region_containing(obj); 986 stringStream obj_region; 987 ro->print_on(&obj_region); 988 989 ShenandoahHeapRegion* rf = heap->heap_region_containing(fwd); 990 stringStream fwd_region; 991 rf->print_on(&fwd_region); 992 993 fatal("forwardee should be self, or another region: " PTR_FORMAT " -> " PTR_FORMAT "\n %s %s", 994 p2i(obj), p2i(fwd), 995 obj_region.as_string(), fwd_region.as_string()); 996 } 997 998 if (!oopDesc::unsafe_equals(obj, fwd)) { 999 oop fwd2 = oop(BrooksPointer::get_raw(fwd)); 1000 if (!oopDesc::unsafe_equals(fwd, fwd2)) { 1001 // We should never be forwarded more than once. 1002 ResourceMark rm; 1003 1004 ShenandoahHeapRegion* ro = heap->heap_region_containing(obj); 1005 stringStream obj_region; 1006 ro->print_on(&obj_region); 1007 1008 ShenandoahHeapRegion* rf = heap->heap_region_containing(fwd); 1009 stringStream fwd_region; 1010 rf->print_on(&fwd_region); 1011 1012 ShenandoahHeapRegion* rf2 = heap->heap_region_containing(fwd2); 1013 stringStream fwd2_region; 1014 rf2->print_on(&fwd2_region); 1015 1016 fatal("Multiple forwardings: " PTR_FORMAT " -> " PTR_FORMAT " -> " PTR_FORMAT "\n %s %s %s", 1017 p2i(obj), p2i(fwd), p2i(fwd2), 1018 obj_region.as_string(), fwd_region.as_string(), fwd2_region.as_string()); 1019 } 1020 } 1021 } 1022 1023 void ShenandoahVerifier::verify_oop(oop obj) { 1024 oop fwd = oop(BrooksPointer::get_raw(obj)); 1025 verify_oop_fwdptr(obj, fwd); 1026 }