42 ShenandoahVerifierStack* _stack;
43 ShenandoahHeap* _heap;
44 MarkBitMap* _map;
45 ShenandoahLivenessData* _ld;
46 void* _interior_loc;
47 oop _loc;
48
49 public:
50 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
51 const char* phase, ShenandoahVerifier::VerifyOptions options) :
52 _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL),
53 _phase(phase), _options(options) {};
54
55 private:
56 void print_obj(ShenandoahMessageBuffer& msg, oop obj) {
57 ShenandoahHeapRegion *r = _heap->heap_region_containing(obj);
58 stringStream ss;
59 r->print_on(&ss);
60
61 msg.append(" " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name());
62 msg.append(" %3s allocated after complete mark start\n", _heap->allocated_after_complete_mark_start((HeapWord *) obj) ? "" : "not");
63 msg.append(" %3s allocated after next mark start\n", _heap->allocated_after_next_mark_start((HeapWord *) obj) ? "" : "not");
64 msg.append(" %3s marked complete\n", _heap->is_marked_complete(obj) ? "" : "not");
65 msg.append(" %3s marked next\n", _heap->is_marked_next(obj) ? "" : "not");
66 msg.append(" %3s in collection set\n", _heap->in_collection_set(obj) ? "" : "not");
67 msg.append(" region: %s", ss.as_string());
68 }
69
70 void print_non_obj(ShenandoahMessageBuffer& msg, void* loc) {
71 msg.append(" outside of Java heap\n");
72 stringStream ss;
73 os::print_location(&ss, (intptr_t) loc, false);
74 msg.append(" %s\n", ss.as_string());
75 }
76
77 void print_obj_safe(ShenandoahMessageBuffer& msg, void* loc) {
78 msg.append(" " PTR_FORMAT " - safe print, no details\n", p2i(loc));
79 if (_heap->is_in(loc)) {
80 ShenandoahHeapRegion* r = _heap->heap_region_containing(loc);
81 if (r != NULL) {
82 stringStream ss;
83 r->print_on(&ss);
84 msg.append(" region: %s", ss.as_string());
85 }
289 verify(_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
290 "Forwardee end should be within the region");
291 }
292
293 verify(_safe_oop, obj, Metaspace::contains(fwd->klass()),
294 "Forwardee klass pointer must go to metaspace");
295 verify(_safe_oop, obj, obj->klass() == fwd->klass(),
296 "Forwardee klass pointer must go to metaspace");
297
298 oop fwd2 = (oop) BrooksPointer::get_raw(fwd);
299 verify(_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2),
300 "Double forwarding");
301 }
302
303 // ------------ obj and fwd are safe at this point --------------
304
305 switch (_options._verify_marked) {
306 case ShenandoahVerifier::_verify_marked_disable:
307 // skip
308 break;
309 case ShenandoahVerifier::_verify_marked_next:
310 verify(_safe_all, obj, _heap->is_marked_next(obj),
311 "Must be marked in next bitmap");
312 break;
313 case ShenandoahVerifier::_verify_marked_complete:
314 verify(_safe_all, obj, _heap->is_marked_complete(obj),
315 "Must be marked in complete bitmap");
316 break;
317 default:
318 assert(false, "Unhandled mark verification");
319 }
320
321 switch (_options._verify_forwarded) {
322 case ShenandoahVerifier::_verify_forwarded_disable:
323 // skip
324 break;
325 case ShenandoahVerifier::_verify_forwarded_none: {
326 verify(_safe_all, obj, oopDesc::unsafe_equals(obj, fwd),
327 "Should not be forwarded");
328 break;
329 }
330 case ShenandoahVerifier::_verify_forwarded_allow: {
331 if (!oopDesc::unsafe_equals(obj, fwd)) {
332 verify(_safe_all, obj, _heap->heap_region_containing(obj) != _heap->heap_region_containing(fwd),
333 "Forwardee should be in another region");
334 }
335 break;
471 verify(r, !r->is_trash(),
472 "Should not have trash regions");
473 break;
474 case ShenandoahVerifier::_verify_regions_nocset:
475 verify(r, !r->is_cset(),
476 "Should not have cset regions");
477 break;
478 case ShenandoahVerifier::_verify_regions_notrash_nocset:
479 verify(r, !r->is_trash(),
480 "Should not have trash regions");
481 verify(r, !r->is_cset(),
482 "Should not have cset regions");
483 break;
484 default:
485 ShouldNotReachHere();
486 }
487
488 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
489 "Capacity should match region size");
490
491 verify(r, r->bottom() <= _heap->complete_top_at_mark_start(r->bottom()),
492 "Region top should not be less than bottom");
493
494 verify(r, _heap->complete_top_at_mark_start(r->bottom()) <= r->top(),
495 "Complete TAMS should not be larger than top");
496
497 verify(r, r->get_live_data_bytes() <= r->capacity(),
498 "Live data cannot be larger than capacity");
499
500 verify(r, r->garbage() <= r->capacity(),
501 "Garbage cannot be larger than capacity");
502
503 verify(r, r->used() <= r->capacity(),
504 "Used cannot be larger than capacity");
505
506 verify(r, r->get_shared_allocs() <= r->capacity(),
507 "Shared alloc count should not be larger than capacity");
508
509 verify(r, r->get_tlab_allocs() <= r->capacity(),
510 "TLAB alloc count should not be larger than capacity");
511
512 verify(r, r->get_gclab_allocs() <= r->capacity(),
513 "GCLAB alloc count should not be larger than capacity");
514
515 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
622 _options);
623
624 while (true) {
625 size_t v = Atomic::add(1u, &_claimed) - 1;
626 if (v < _heap->num_regions()) {
627 ShenandoahHeapRegion* r = _regions->get(v);
628 if (!r->is_humongous() && !r->is_trash()) {
629 work_regular(r, stack, cl);
630 } else if (r->is_humongous_start()) {
631 work_humongous(r, stack, cl);
632 }
633 } else {
634 break;
635 }
636 }
637 }
638
639 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
640 size_t processed = 0;
641 HeapWord* obj = r->bottom() + BrooksPointer::word_size();
642 if (_heap->is_marked_complete((oop)obj)) {
643 verify_and_follow(obj, stack, cl, &processed);
644 }
645 Atomic::add(processed, &_processed);
646 }
647
648 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
649 size_t processed = 0;
650 MarkBitMap* mark_bit_map = _heap->complete_mark_bit_map();
651 HeapWord* tams = _heap->complete_top_at_mark_start(r->bottom());
652
653 // Bitmaps, before TAMS
654 if (tams > r->bottom()) {
655 HeapWord* start = r->bottom() + BrooksPointer::word_size();
656 HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams);
657
658 while (addr < tams) {
659 verify_and_follow(addr, stack, cl, &processed);
660 addr += BrooksPointer::word_size();
661 if (addr < tams) {
662 addr = mark_bit_map->getNextMarkedWordAddress(addr, tams);
663 }
664 }
665 }
666
667 // Size-based, after TAMS
668 {
669 HeapWord* limit = r->top();
670 HeapWord* addr = tams + BrooksPointer::word_size();
671
751 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
752 // This verifies what application can see, since it only cares about reachable objects.
753 size_t count_reachable = 0;
754 if (ShenandoahVerifyLevel >= 2) {
755 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(),
756 ShenandoahPhaseTimings::_num_phases); // no need for stats
757
758 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options);
759 _heap->workers()->run_task(&task);
760 count_reachable = task.processed();
761 }
762
763 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
764 // not the application, can see during the region scans. There is no reason to process the objects
765 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
766 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
767 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
768 // version
769
770 size_t count_marked = 0;
771 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
772 ShenandoahVerifierMarkedRegionTask task(_heap->regions(), _verification_bit_map, ld, label, options);
773 _heap->workers()->run_task(&task);
774 count_marked = task.processed();
775 } else {
776 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_next || marked == _verify_marked_disable, "Should be");
777 }
778
779 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
780 // marked objects.
781
782 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
783 ShenandoahHeapRegionSet* set = _heap->regions();
784 for (size_t i = 0; i < _heap->num_regions(); i++) {
785 ShenandoahHeapRegion* r = set->get(i);
786
787 juint verf_live = 0;
788 if (r->is_humongous()) {
789 // For humongous objects, test if start region is marked live, and if so,
790 // all humongous regions in that chain have live data equal to their "used".
791 juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]);
792 if (start_live > 0) {
793 verf_live = (juint)(r->used() / HeapWordSize);
794 }
795 } else {
796 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]);
797 }
798
799 size_t reg_live = r->get_live_data_words();
800 if (reg_live != verf_live) {
801 ResourceMark rm;
802 stringStream ss;
836 _verify_liveness_disable, // no reliable liveness data
837 _verify_regions_notrash // no trash regions
838 );
839 } else {
840 verify_at_safepoint(
841 "Before Mark",
842 _verify_forwarded_none, // UR should have fixed up
843 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
844 _verify_matrix_conservative, // UR should have fixed matrix
845 _verify_cset_none, // UR should have fixed this
846 _verify_liveness_disable, // no reliable liveness data
847 _verify_regions_notrash // no trash regions
848 );
849 }
850 }
851
852 void ShenandoahVerifier::verify_after_concmark() {
853 verify_at_safepoint(
854 "After Mark",
855 _verify_forwarded_none, // no forwarded references
856 _verify_marked_complete, // bitmaps as precise as we can get
857 _verify_matrix_disable, // matrix might be foobared
858 _verify_cset_none, // no references to cset anymore
859 _verify_liveness_complete, // liveness data must be complete here
860 _verify_regions_disable // trash regions not yet recycled
861 );
862 }
863
864 void ShenandoahVerifier::verify_before_evacuation() {
865 // Evacuation is always preceded by mark, but we want to have a sanity check after
866 // selecting the collection set, and (immediate) regions recycling
867 verify_at_safepoint(
868 "Before Evacuation",
869 _verify_forwarded_none, // no forwarded references
870 _verify_marked_complete, // walk over marked objects too
871 _verify_matrix_disable, // skip, verified after mark
872 _verify_cset_disable, // skip, verified after mark
873 _verify_liveness_disable, // skip, verified after mark
874 _verify_regions_disable // trash regions not yet recycled
875 );
876 }
877
878 void ShenandoahVerifier::verify_after_evacuation() {
879 verify_at_safepoint(
880 "After Evacuation",
881 _verify_forwarded_allow, // objects are still forwarded
882 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
883 _verify_matrix_disable, // matrix is inconsistent here
884 _verify_cset_forwarded, // all cset refs are fully forwarded
885 _verify_liveness_disable, // no reliable liveness data anymore
886 _verify_regions_notrash // trash regions have been recycled already
887 );
888 }
889
890 void ShenandoahVerifier::verify_before_updaterefs() {
891 verify_at_safepoint(
892 "Before Updating References",
893 _verify_forwarded_allow, // forwarded references allowed
894 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
895 _verify_matrix_disable, // matrix is inconsistent here
896 _verify_cset_forwarded, // all cset refs are fully forwarded
897 _verify_liveness_disable, // no reliable liveness data anymore
898 _verify_regions_notrash // trash regions have been recycled already
899 );
900 }
901
902 void ShenandoahVerifier::verify_after_updaterefs() {
903 verify_at_safepoint(
904 "After Updating References",
905 _verify_forwarded_none, // no forwarded references
906 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
907 _verify_matrix_conservative, // matrix is conservatively consistent
908 _verify_cset_none, // no cset references, all updated
909 _verify_liveness_disable, // no reliable liveness data anymore
910 _verify_regions_nocset // no cset regions, trash regions have appeared
911 );
912 }
913
914 void ShenandoahVerifier::verify_before_partial() {
915 verify_at_safepoint(
916 "Before Partial GC",
917 _verify_forwarded_none, // cannot have forwarded objects
918 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
919 _verify_matrix_conservative, // matrix is conservatively consistent
920 _verify_cset_none, // no cset references before partial
921 _verify_liveness_disable, // no reliable liveness data anymore
922 _verify_regions_notrash_nocset // no trash and no cset regions
923 );
924 }
925
926 void ShenandoahVerifier::verify_after_partial() {
927 verify_at_safepoint(
928 "After Partial GC",
929 _verify_forwarded_none, // cannot have forwarded objects
930 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
931 _verify_matrix_conservative, // matrix is conservatively consistent
932 _verify_cset_none, // no cset references left after partial
933 _verify_liveness_disable, // no reliable liveness data anymore
934 _verify_regions_nocset // no cset regions, trash regions allowed
935 );
936 }
937
938 void ShenandoahVerifier::verify_before_fullgc() {
939 verify_at_safepoint(
940 "Before Full GC",
941 _verify_forwarded_allow, // can have forwarded objects
942 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
943 _verify_matrix_disable, // matrix might be foobared
944 _verify_cset_disable, // cset might be foobared
945 _verify_liveness_disable, // no reliable liveness data anymore
946 _verify_regions_disable // no reliable region data here
947 );
948 }
949
950 void ShenandoahVerifier::verify_after_fullgc() {
951 verify_at_safepoint(
952 "After Full GC",
953 _verify_forwarded_none, // all objects are non-forwarded
954 _verify_marked_complete, // all objects are marked in complete bitmap
955 _verify_matrix_conservative, // matrix is conservatively consistent
956 _verify_cset_none, // no cset references
957 _verify_liveness_disable, // no reliable liveness data anymore
958 _verify_regions_notrash_nocset // no trash, no cset
959 );
960 }
961
962 void ShenandoahVerifier::verify_oop_fwdptr(oop obj, oop fwd) {
963 guarantee(UseShenandoahGC, "must only be called when Shenandoah is used");
964
965 ShenandoahHeap* heap = ShenandoahHeap::heap();
966
967 guarantee(obj != NULL, "oop is not NULL");
968 guarantee(heap->is_in(obj), "oop must point to a heap address: " PTR_FORMAT, p2i(obj));
969
970 guarantee(fwd != NULL, "forwardee is not NULL");
971 if (!heap->is_in(fwd)) {
972 ResourceMark rm;
973 ShenandoahHeapRegion* r = heap->heap_region_containing(obj);
974 stringStream obj_region;
|
42 ShenandoahVerifierStack* _stack;
43 ShenandoahHeap* _heap;
44 MarkBitMap* _map;
45 ShenandoahLivenessData* _ld;
46 void* _interior_loc;
47 oop _loc;
48
49 public:
50 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
51 const char* phase, ShenandoahVerifier::VerifyOptions options) :
52 _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL),
53 _phase(phase), _options(options) {};
54
55 private:
56 void print_obj(ShenandoahMessageBuffer& msg, oop obj) {
57 ShenandoahHeapRegion *r = _heap->heap_region_containing(obj);
58 stringStream ss;
59 r->print_on(&ss);
60
61 msg.append(" " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name());
62 msg.append(" %3s allocated after mark start\n", _heap->allocated_after_mark_start((HeapWord *) obj) ? "" : "not");
63 msg.append(" %3s marked\n", _heap->is_marked(obj) ? "" : "not");
64 msg.append(" %3s in collection set\n", _heap->in_collection_set(obj) ? "" : "not");
65 msg.append(" region: %s", ss.as_string());
66 }
67
68 void print_non_obj(ShenandoahMessageBuffer& msg, void* loc) {
69 msg.append(" outside of Java heap\n");
70 stringStream ss;
71 os::print_location(&ss, (intptr_t) loc, false);
72 msg.append(" %s\n", ss.as_string());
73 }
74
75 void print_obj_safe(ShenandoahMessageBuffer& msg, void* loc) {
76 msg.append(" " PTR_FORMAT " - safe print, no details\n", p2i(loc));
77 if (_heap->is_in(loc)) {
78 ShenandoahHeapRegion* r = _heap->heap_region_containing(loc);
79 if (r != NULL) {
80 stringStream ss;
81 r->print_on(&ss);
82 msg.append(" region: %s", ss.as_string());
83 }
287 verify(_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
288 "Forwardee end should be within the region");
289 }
290
291 verify(_safe_oop, obj, Metaspace::contains(fwd->klass()),
292 "Forwardee klass pointer must go to metaspace");
293 verify(_safe_oop, obj, obj->klass() == fwd->klass(),
294 "Forwardee klass pointer must go to metaspace");
295
296 oop fwd2 = (oop) BrooksPointer::get_raw(fwd);
297 verify(_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2),
298 "Double forwarding");
299 }
300
301 // ------------ obj and fwd are safe at this point --------------
302
303 switch (_options._verify_marked) {
304 case ShenandoahVerifier::_verify_marked_disable:
305 // skip
306 break;
307 case ShenandoahVerifier::_verify_marked:
308 verify(_safe_all, obj, _heap->is_marked(obj),
309 "Must be marked");
310 break;
311 default:
312 assert(false, "Unhandled mark verification");
313 }
314
315 switch (_options._verify_forwarded) {
316 case ShenandoahVerifier::_verify_forwarded_disable:
317 // skip
318 break;
319 case ShenandoahVerifier::_verify_forwarded_none: {
320 verify(_safe_all, obj, oopDesc::unsafe_equals(obj, fwd),
321 "Should not be forwarded");
322 break;
323 }
324 case ShenandoahVerifier::_verify_forwarded_allow: {
325 if (!oopDesc::unsafe_equals(obj, fwd)) {
326 verify(_safe_all, obj, _heap->heap_region_containing(obj) != _heap->heap_region_containing(fwd),
327 "Forwardee should be in another region");
328 }
329 break;
465 verify(r, !r->is_trash(),
466 "Should not have trash regions");
467 break;
468 case ShenandoahVerifier::_verify_regions_nocset:
469 verify(r, !r->is_cset(),
470 "Should not have cset regions");
471 break;
472 case ShenandoahVerifier::_verify_regions_notrash_nocset:
473 verify(r, !r->is_trash(),
474 "Should not have trash regions");
475 verify(r, !r->is_cset(),
476 "Should not have cset regions");
477 break;
478 default:
479 ShouldNotReachHere();
480 }
481
482 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
483 "Capacity should match region size");
484
485 verify(r, r->bottom() <= _heap->top_at_mark_start(r->bottom()),
486 "Region top should not be less than bottom");
487
488 verify(r, _heap->top_at_mark_start(r->bottom()) <= r->top(),
489 "TAMS should not be larger than top");
490
491 verify(r, r->get_live_data_bytes() <= r->capacity(),
492 "Live data cannot be larger than capacity");
493
494 verify(r, r->garbage() <= r->capacity(),
495 "Garbage cannot be larger than capacity");
496
497 verify(r, r->used() <= r->capacity(),
498 "Used cannot be larger than capacity");
499
500 verify(r, r->get_shared_allocs() <= r->capacity(),
501 "Shared alloc count should not be larger than capacity");
502
503 verify(r, r->get_tlab_allocs() <= r->capacity(),
504 "TLAB alloc count should not be larger than capacity");
505
506 verify(r, r->get_gclab_allocs() <= r->capacity(),
507 "GCLAB alloc count should not be larger than capacity");
508
509 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
616 _options);
617
618 while (true) {
619 size_t v = Atomic::add(1u, &_claimed) - 1;
620 if (v < _heap->num_regions()) {
621 ShenandoahHeapRegion* r = _regions->get(v);
622 if (!r->is_humongous() && !r->is_trash()) {
623 work_regular(r, stack, cl);
624 } else if (r->is_humongous_start()) {
625 work_humongous(r, stack, cl);
626 }
627 } else {
628 break;
629 }
630 }
631 }
632
633 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
634 size_t processed = 0;
635 HeapWord* obj = r->bottom() + BrooksPointer::word_size();
636 if (_heap->is_marked((oop)obj)) {
637 verify_and_follow(obj, stack, cl, &processed);
638 }
639 Atomic::add(processed, &_processed);
640 }
641
642 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
643 size_t processed = 0;
644 MarkBitMap* mark_bit_map = _heap->mark_bit_map();
645 HeapWord* tams = _heap->top_at_mark_start(r->bottom());
646
647 // Bitmaps, before TAMS
648 if (tams > r->bottom()) {
649 HeapWord* start = r->bottom() + BrooksPointer::word_size();
650 HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams);
651
652 while (addr < tams) {
653 verify_and_follow(addr, stack, cl, &processed);
654 addr += BrooksPointer::word_size();
655 if (addr < tams) {
656 addr = mark_bit_map->getNextMarkedWordAddress(addr, tams);
657 }
658 }
659 }
660
661 // Size-based, after TAMS
662 {
663 HeapWord* limit = r->top();
664 HeapWord* addr = tams + BrooksPointer::word_size();
665
745 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
746 // This verifies what application can see, since it only cares about reachable objects.
747 size_t count_reachable = 0;
748 if (ShenandoahVerifyLevel >= 2) {
749 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(),
750 ShenandoahPhaseTimings::_num_phases); // no need for stats
751
752 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options);
753 _heap->workers()->run_task(&task);
754 count_reachable = task.processed();
755 }
756
757 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
758 // not the application, can see during the region scans. There is no reason to process the objects
759 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
760 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
761 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
762 // version
763
764 size_t count_marked = 0;
765 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked) {
766 ShenandoahVerifierMarkedRegionTask task(_heap->regions(), _verification_bit_map, ld, label, options);
767 _heap->workers()->run_task(&task);
768 count_marked = task.processed();
769 } else {
770 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_disable, "Should be");
771 }
772
773 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
774 // marked objects.
775
776 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked && liveness == _verify_liveness_complete) {
777 ShenandoahHeapRegionSet* set = _heap->regions();
778 for (size_t i = 0; i < _heap->num_regions(); i++) {
779 ShenandoahHeapRegion* r = set->get(i);
780
781 juint verf_live = 0;
782 if (r->is_humongous()) {
783 // For humongous objects, test if start region is marked live, and if so,
784 // all humongous regions in that chain have live data equal to their "used".
785 juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]);
786 if (start_live > 0) {
787 verf_live = (juint)(r->used() / HeapWordSize);
788 }
789 } else {
790 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]);
791 }
792
793 size_t reg_live = r->get_live_data_words();
794 if (reg_live != verf_live) {
795 ResourceMark rm;
796 stringStream ss;
830 _verify_liveness_disable, // no reliable liveness data
831 _verify_regions_notrash // no trash regions
832 );
833 } else {
834 verify_at_safepoint(
835 "Before Mark",
836 _verify_forwarded_none, // UR should have fixed up
837 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
838 _verify_matrix_conservative, // UR should have fixed matrix
839 _verify_cset_none, // UR should have fixed this
840 _verify_liveness_disable, // no reliable liveness data
841 _verify_regions_notrash // no trash regions
842 );
843 }
844 }
845
846 void ShenandoahVerifier::verify_after_concmark() {
847 verify_at_safepoint(
848 "After Mark",
849 _verify_forwarded_none, // no forwarded references
850 _verify_marked, // bitmaps as precise as we can get
851 _verify_matrix_disable, // matrix might be foobared
852 _verify_cset_none, // no references to cset anymore
853 _verify_liveness_complete, // liveness data must be complete here
854 _verify_regions_disable // trash regions not yet recycled
855 );
856 }
857
858 void ShenandoahVerifier::verify_before_evacuation() {
859 // Evacuation is always preceded by mark, but we want to have a sanity check after
860 // selecting the collection set, and (immediate) regions recycling
861 verify_at_safepoint(
862 "Before Evacuation",
863 _verify_forwarded_none, // no forwarded references
864 _verify_marked, // walk over marked objects too
865 _verify_matrix_disable, // skip, verified after mark
866 _verify_cset_disable, // skip, verified after mark
867 _verify_liveness_disable, // skip, verified after mark
868 _verify_regions_disable // trash regions not yet recycled
869 );
870 }
871
872 void ShenandoahVerifier::verify_after_evacuation() {
873 verify_at_safepoint(
874 "After Evacuation",
875 _verify_forwarded_allow, // objects are still forwarded
876 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well
877 _verify_matrix_disable, // matrix is inconsistent here
878 _verify_cset_forwarded, // all cset refs are fully forwarded
879 _verify_liveness_disable, // no reliable liveness data anymore
880 _verify_regions_notrash // trash regions have been recycled already
881 );
882 }
883
884 void ShenandoahVerifier::verify_before_updaterefs() {
885 verify_at_safepoint(
886 "Before Updating References",
887 _verify_forwarded_allow, // forwarded references allowed
888 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well
889 _verify_matrix_disable, // matrix is inconsistent here
890 _verify_cset_forwarded, // all cset refs are fully forwarded
891 _verify_liveness_disable, // no reliable liveness data anymore
892 _verify_regions_notrash // trash regions have been recycled already
893 );
894 }
895
896 void ShenandoahVerifier::verify_after_updaterefs() {
897 verify_at_safepoint(
898 "After Updating References",
899 _verify_forwarded_none, // no forwarded references
900 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well
901 _verify_matrix_conservative, // matrix is conservatively consistent
902 _verify_cset_none, // no cset references, all updated
903 _verify_liveness_disable, // no reliable liveness data anymore
904 _verify_regions_nocset // no cset regions, trash regions have appeared
905 );
906 }
907
908 void ShenandoahVerifier::verify_before_partial() {
909 verify_at_safepoint(
910 "Before Partial GC",
911 _verify_forwarded_none, // cannot have forwarded objects
912 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well
913 _verify_matrix_conservative, // matrix is conservatively consistent
914 _verify_cset_none, // no cset references before partial
915 _verify_liveness_disable, // no reliable liveness data anymore
916 _verify_regions_notrash_nocset // no trash and no cset regions
917 );
918 }
919
920 void ShenandoahVerifier::verify_after_partial() {
921 verify_at_safepoint(
922 "After Partial GC",
923 _verify_forwarded_none, // cannot have forwarded objects
924 _verify_marked, // bitmaps might be stale, but alloc-after-mark should be well
925 _verify_matrix_conservative, // matrix is conservatively consistent
926 _verify_cset_none, // no cset references left after partial
927 _verify_liveness_disable, // no reliable liveness data anymore
928 _verify_regions_nocset // no cset regions, trash regions allowed
929 );
930 }
931
932 void ShenandoahVerifier::verify_before_fullgc() {
933 verify_at_safepoint(
934 "Before Full GC",
935 _verify_forwarded_allow, // can have forwarded objects
936 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
937 _verify_matrix_disable, // matrix might be foobared
938 _verify_cset_disable, // cset might be foobared
939 _verify_liveness_disable, // no reliable liveness data anymore
940 _verify_regions_disable // no reliable region data here
941 );
942 }
943
944 void ShenandoahVerifier::verify_after_fullgc() {
945 verify_at_safepoint(
946 "After Full GC",
947 _verify_forwarded_none, // all objects are non-forwarded
948 _verify_marked, // all objects are marked
949 _verify_matrix_conservative, // matrix is conservatively consistent
950 _verify_cset_none, // no cset references
951 _verify_liveness_disable, // no reliable liveness data anymore
952 _verify_regions_notrash_nocset // no trash, no cset
953 );
954 }
955
956 void ShenandoahVerifier::verify_oop_fwdptr(oop obj, oop fwd) {
957 guarantee(UseShenandoahGC, "must only be called when Shenandoah is used");
958
959 ShenandoahHeap* heap = ShenandoahHeap::heap();
960
961 guarantee(obj != NULL, "oop is not NULL");
962 guarantee(heap->is_in(obj), "oop must point to a heap address: " PTR_FORMAT, p2i(obj));
963
964 guarantee(fwd != NULL, "forwardee is not NULL");
965 if (!heap->is_in(fwd)) {
966 ResourceMark rm;
967 ShenandoahHeapRegion* r = heap->heap_region_containing(obj);
968 stringStream obj_region;
|