< prev index next >

src/share/vm/gc/shenandoah/shenandoahVerifier.cpp

Print this page
rev 14452 : [mq]: onebitmap.patch


  42   ShenandoahVerifierStack* _stack;
  43   ShenandoahHeap* _heap;
  44   MarkBitMap* _map;
  45   ShenandoahLivenessData* _ld;
  46   void* _interior_loc;
  47   oop _loc;
  48 
  49 public:
  50   ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
  51                              const char* phase, ShenandoahVerifier::VerifyOptions options) :
  52           _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL),
  53           _phase(phase), _options(options) {};
  54 
  55 private:
  56   void print_obj(ShenandoahMessageBuffer& msg, oop obj) {
  57     ShenandoahHeapRegion *r = _heap->heap_region_containing(obj);
  58     stringStream ss;
  59     r->print_on(&ss);
  60 
  61     msg.append("  " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name());
  62     msg.append("    %3s allocated after complete mark start\n", _heap->allocated_after_complete_mark_start((HeapWord *) obj) ? "" : "not");
  63     msg.append("    %3s allocated after next mark start\n",     _heap->allocated_after_next_mark_start((HeapWord *) obj)     ? "" : "not");
  64     msg.append("    %3s marked complete\n",      _heap->is_marked_complete(obj) ? "" : "not");
  65     msg.append("    %3s marked next\n",          _heap->is_marked_next(obj) ? "" : "not");
  66     msg.append("    %3s in collection set\n",    _heap->in_collection_set(obj) ? "" : "not");
  67     msg.append("  region: %s", ss.as_string());
  68   }
  69 
  70   void print_non_obj(ShenandoahMessageBuffer& msg, void* loc) {
  71     msg.append("  outside of Java heap\n");
  72     stringStream ss;
  73     os::print_location(&ss, (intptr_t) loc, false);
  74     msg.append("  %s\n", ss.as_string());
  75   }
  76 
  77   void print_obj_safe(ShenandoahMessageBuffer& msg, void* loc) {
  78     msg.append("  " PTR_FORMAT " - safe print, no details\n", p2i(loc));
  79     if (_heap->is_in(loc)) {
  80       ShenandoahHeapRegion* r = _heap->heap_region_containing(loc);
  81       if (r != NULL) {
  82         stringStream ss;
  83         r->print_on(&ss);
  84         msg.append("  region: %s", ss.as_string());
  85       }


 289         verify(_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
 290                "Forwardee end should be within the region");
 291       }
 292 
 293       verify(_safe_oop, obj, Metaspace::contains(fwd->klass()),
 294              "Forwardee klass pointer must go to metaspace");
 295       verify(_safe_oop, obj, obj->klass() == fwd->klass(),
 296              "Forwardee klass pointer must go to metaspace");
 297 
 298       oop fwd2 = (oop) BrooksPointer::get_raw(fwd);
 299       verify(_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2),
 300              "Double forwarding");
 301     }
 302 
 303     // ------------ obj and fwd are safe at this point --------------
 304 
 305     switch (_options._verify_marked) {
 306       case ShenandoahVerifier::_verify_marked_disable:
 307         // skip
 308         break;
 309       case ShenandoahVerifier::_verify_marked_next:
 310         verify(_safe_all, obj, _heap->is_marked_next(obj),
 311                "Must be marked in next bitmap");
 312         break;
 313       case ShenandoahVerifier::_verify_marked_complete:
 314         verify(_safe_all, obj, _heap->is_marked_complete(obj),
 315                "Must be marked in complete bitmap");
 316         break;
 317       default:
 318         assert(false, "Unhandled mark verification");
 319     }
 320 
 321     switch (_options._verify_forwarded) {
 322       case ShenandoahVerifier::_verify_forwarded_disable:
 323         // skip
 324         break;
 325       case ShenandoahVerifier::_verify_forwarded_none: {
 326         verify(_safe_all, obj, oopDesc::unsafe_equals(obj, fwd),
 327                "Should not be forwarded");
 328         break;
 329       }
 330       case ShenandoahVerifier::_verify_forwarded_allow: {
 331         if (!oopDesc::unsafe_equals(obj, fwd)) {
 332           verify(_safe_all, obj, _heap->heap_region_containing(obj) != _heap->heap_region_containing(fwd),
 333                  "Forwardee should be in another region");
 334         }
 335         break;


 471         verify(r, !r->is_trash(),
 472                "Should not have trash regions");
 473         break;
 474       case ShenandoahVerifier::_verify_regions_nocset:
 475         verify(r, !r->is_cset(),
 476                "Should not have cset regions");
 477         break;
 478       case ShenandoahVerifier::_verify_regions_notrash_nocset:
 479         verify(r, !r->is_trash(),
 480                "Should not have trash regions");
 481         verify(r, !r->is_cset(),
 482                "Should not have cset regions");
 483         break;
 484       default:
 485         ShouldNotReachHere();
 486     }
 487 
 488     verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
 489            "Capacity should match region size");
 490 
 491     verify(r, r->bottom() <= _heap->complete_top_at_mark_start(r->bottom()),
 492            "Region top should not be less than bottom");
 493 
 494     verify(r, _heap->complete_top_at_mark_start(r->bottom()) <= r->top(),
 495            "Complete TAMS should not be larger than top");
 496 
 497     verify(r, r->get_live_data_bytes() <= r->capacity(),
 498            "Live data cannot be larger than capacity");
 499 
 500     verify(r, r->garbage() <= r->capacity(),
 501            "Garbage cannot be larger than capacity");
 502 
 503     verify(r, r->used() <= r->capacity(),
 504            "Used cannot be larger than capacity");
 505 
 506     verify(r, r->get_shared_allocs() <= r->capacity(),
 507            "Shared alloc count should not be larger than capacity");
 508 
 509     verify(r, r->get_tlab_allocs() <= r->capacity(),
 510            "TLAB alloc count should not be larger than capacity");
 511 
 512     verify(r, r->get_gclab_allocs() <= r->capacity(),
 513            "GCLAB alloc count should not be larger than capacity");
 514 
 515     verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),


 622                                   _options);
 623 
 624     while (true) {
 625       size_t v = Atomic::add(1u, &_claimed) - 1;
 626       if (v < _heap->num_regions()) {
 627         ShenandoahHeapRegion* r = _regions->get(v);
 628         if (!r->is_humongous() && !r->is_trash()) {
 629           work_regular(r, stack, cl);
 630         } else if (r->is_humongous_start()) {
 631           work_humongous(r, stack, cl);
 632         }
 633       } else {
 634         break;
 635       }
 636     }
 637   }
 638 
 639   virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
 640     size_t processed = 0;
 641     HeapWord* obj = r->bottom() + BrooksPointer::word_size();
 642     if (_heap->is_marked_complete((oop)obj)) {
 643       verify_and_follow(obj, stack, cl, &processed);
 644     }
 645     Atomic::add(processed, &_processed);
 646   }
 647 
 648   virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
 649     size_t processed = 0;
 650     MarkBitMap* mark_bit_map = _heap->complete_mark_bit_map();
 651     HeapWord* tams = _heap->complete_top_at_mark_start(r->bottom());
 652 
 653     // Bitmaps, before TAMS
 654     if (tams > r->bottom()) {
 655       HeapWord* start = r->bottom() + BrooksPointer::word_size();
 656       HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams);
 657 
 658       while (addr < tams) {
 659         verify_and_follow(addr, stack, cl, &processed);
 660         addr += BrooksPointer::word_size();
 661         if (addr < tams) {
 662           addr = mark_bit_map->getNextMarkedWordAddress(addr, tams);
 663         }
 664       }
 665     }
 666 
 667     // Size-based, after TAMS
 668     {
 669       HeapWord* limit = r->top();
 670       HeapWord* addr = tams + BrooksPointer::word_size();
 671 


 751   // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
 752   // This verifies what application can see, since it only cares about reachable objects.
 753   size_t count_reachable = 0;
 754   if (ShenandoahVerifyLevel >= 2) {
 755     ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(),
 756                                ShenandoahPhaseTimings::_num_phases); // no need for stats
 757 
 758     ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options);
 759     _heap->workers()->run_task(&task);
 760     count_reachable = task.processed();
 761   }
 762 
 763   // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
 764   // not the application, can see during the region scans. There is no reason to process the objects
 765   // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
 766   // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
 767   // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
 768   // version
 769 
 770   size_t count_marked = 0;
 771   if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
 772     ShenandoahVerifierMarkedRegionTask task(_heap->regions(), _verification_bit_map, ld, label, options);
 773     _heap->workers()->run_task(&task);
 774     count_marked = task.processed();
 775   } else {
 776     guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_next || marked == _verify_marked_disable, "Should be");
 777   }
 778 
 779   // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
 780   // marked objects.
 781 
 782   if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
 783     ShenandoahHeapRegionSet* set = _heap->regions();
 784     for (size_t i = 0; i < _heap->num_regions(); i++) {
 785       ShenandoahHeapRegion* r = set->get(i);
 786 
 787       juint verf_live = 0;
 788       if (r->is_humongous()) {
 789         // For humongous objects, test if start region is marked live, and if so,
 790         // all humongous regions in that chain have live data equal to their "used".
 791         juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]);
 792         if (start_live > 0) {
 793           verf_live = (juint)(r->used() / HeapWordSize);
 794         }
 795       } else {
 796         verf_live = OrderAccess::load_acquire(&ld[r->region_number()]);
 797       }
 798 
 799       size_t reg_live = r->get_live_data_words();
 800       if (reg_live != verf_live) {
 801         ResourceMark rm;
 802         stringStream ss;


 836             _verify_liveness_disable,    // no reliable liveness data
 837             _verify_regions_notrash      // no trash regions
 838     );
 839   } else {
 840     verify_at_safepoint(
 841             "Before Mark",
 842             _verify_forwarded_none,      // UR should have fixed up
 843             _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
 844             _verify_matrix_conservative, // UR should have fixed matrix
 845             _verify_cset_none,           // UR should have fixed this
 846             _verify_liveness_disable,    // no reliable liveness data
 847             _verify_regions_notrash      // no trash regions
 848     );
 849   }
 850 }
 851 
 852 void ShenandoahVerifier::verify_after_concmark() {
 853   verify_at_safepoint(
 854           "After Mark",
 855           _verify_forwarded_none,      // no forwarded references
 856           _verify_marked_complete,     // bitmaps as precise as we can get
 857           _verify_matrix_disable,      // matrix might be foobared
 858           _verify_cset_none,           // no references to cset anymore
 859           _verify_liveness_complete,   // liveness data must be complete here
 860           _verify_regions_disable      // trash regions not yet recycled
 861   );
 862 }
 863 
 864 void ShenandoahVerifier::verify_before_evacuation() {
 865   // Evacuation is always preceded by mark, but we want to have a sanity check after
 866   // selecting the collection set, and (immediate) regions recycling
 867   verify_at_safepoint(
 868           "Before Evacuation",
 869           _verify_forwarded_none,    // no forwarded references
 870           _verify_marked_complete,   // walk over marked objects too
 871           _verify_matrix_disable,    // skip, verified after mark
 872           _verify_cset_disable,      // skip, verified after mark
 873           _verify_liveness_disable,  // skip, verified after mark
 874           _verify_regions_disable    // trash regions not yet recycled
 875   );
 876 }
 877 
 878 void ShenandoahVerifier::verify_after_evacuation() {
 879   verify_at_safepoint(
 880           "After Evacuation",
 881           _verify_forwarded_allow,     // objects are still forwarded
 882           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
 883           _verify_matrix_disable,      // matrix is inconsistent here
 884           _verify_cset_forwarded,      // all cset refs are fully forwarded
 885           _verify_liveness_disable,    // no reliable liveness data anymore
 886           _verify_regions_notrash      // trash regions have been recycled already
 887   );
 888 }
 889 
 890 void ShenandoahVerifier::verify_before_updaterefs() {
 891   verify_at_safepoint(
 892           "Before Updating References",
 893           _verify_forwarded_allow,     // forwarded references allowed
 894           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
 895           _verify_matrix_disable,      // matrix is inconsistent here
 896           _verify_cset_forwarded,      // all cset refs are fully forwarded
 897           _verify_liveness_disable,    // no reliable liveness data anymore
 898           _verify_regions_notrash      // trash regions have been recycled already
 899   );
 900 }
 901 
 902 void ShenandoahVerifier::verify_after_updaterefs() {
 903   verify_at_safepoint(
 904           "After Updating References",
 905           _verify_forwarded_none,      // no forwarded references
 906           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
 907           _verify_matrix_conservative, // matrix is conservatively consistent
 908           _verify_cset_none,           // no cset references, all updated
 909           _verify_liveness_disable,    // no reliable liveness data anymore
 910           _verify_regions_nocset       // no cset regions, trash regions have appeared
 911   );
 912 }
 913 
 914 void ShenandoahVerifier::verify_before_partial() {
 915   verify_at_safepoint(
 916           "Before Partial GC",
 917           _verify_forwarded_none,      // cannot have forwarded objects
 918           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
 919           _verify_matrix_conservative, // matrix is conservatively consistent
 920           _verify_cset_none,           // no cset references before partial
 921           _verify_liveness_disable,    // no reliable liveness data anymore
 922           _verify_regions_notrash_nocset // no trash and no cset regions
 923   );
 924 }
 925 
 926 void ShenandoahVerifier::verify_after_partial() {
 927   verify_at_safepoint(
 928           "After Partial GC",
 929           _verify_forwarded_none,      // cannot have forwarded objects
 930           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
 931           _verify_matrix_conservative, // matrix is conservatively consistent
 932           _verify_cset_none,           // no cset references left after partial
 933           _verify_liveness_disable,    // no reliable liveness data anymore
 934           _verify_regions_nocset       // no cset regions, trash regions allowed
 935   );
 936 }
 937 
 938 void ShenandoahVerifier::verify_before_fullgc() {
 939   verify_at_safepoint(
 940           "Before Full GC",
 941           _verify_forwarded_allow,     // can have forwarded objects
 942           _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
 943           _verify_matrix_disable,      // matrix might be foobared
 944           _verify_cset_disable,        // cset might be foobared
 945           _verify_liveness_disable,    // no reliable liveness data anymore
 946           _verify_regions_disable      // no reliable region data here
 947   );
 948 }
 949 
 950 void ShenandoahVerifier::verify_after_fullgc() {
 951   verify_at_safepoint(
 952           "After Full GC",
 953           _verify_forwarded_none,      // all objects are non-forwarded
 954           _verify_marked_complete,     // all objects are marked in complete bitmap
 955           _verify_matrix_conservative, // matrix is conservatively consistent
 956           _verify_cset_none,           // no cset references
 957           _verify_liveness_disable,    // no reliable liveness data anymore
 958           _verify_regions_notrash_nocset // no trash, no cset
 959   );
 960 }
 961 
 962 void ShenandoahVerifier::verify_oop_fwdptr(oop obj, oop fwd) {
 963   guarantee(UseShenandoahGC, "must only be called when Shenandoah is used");
 964 
 965   ShenandoahHeap* heap = ShenandoahHeap::heap();
 966 
 967   guarantee(obj != NULL, "oop is not NULL");
 968   guarantee(heap->is_in(obj), "oop must point to a heap address: " PTR_FORMAT, p2i(obj));
 969 
 970   guarantee(fwd != NULL, "forwardee is not NULL");
 971   if (!heap->is_in(fwd)) {
 972     ResourceMark rm;
 973     ShenandoahHeapRegion* r = heap->heap_region_containing(obj);
 974     stringStream obj_region;




  42   ShenandoahVerifierStack* _stack;
  43   ShenandoahHeap* _heap;
  44   MarkBitMap* _map;
  45   ShenandoahLivenessData* _ld;
  46   void* _interior_loc;
  47   oop _loc;
  48 
  49 public:
  50   ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
  51                              const char* phase, ShenandoahVerifier::VerifyOptions options) :
  52           _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL),
  53           _phase(phase), _options(options) {};
  54 
  55 private:
  56   void print_obj(ShenandoahMessageBuffer& msg, oop obj) {
  57     ShenandoahHeapRegion *r = _heap->heap_region_containing(obj);
  58     stringStream ss;
  59     r->print_on(&ss);
  60 
  61     msg.append("  " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name());
  62     msg.append("    %3s allocated after mark start\n", _heap->allocated_after_mark_start((HeapWord *) obj) ? "" : "not");
  63     msg.append("    %3s marked\n",            _heap->is_marked(obj) ? "" : "not");


  64     msg.append("    %3s in collection set\n", _heap->in_collection_set(obj) ? "" : "not");
  65     msg.append("  region: %s", ss.as_string());
  66   }
  67 
  68   void print_non_obj(ShenandoahMessageBuffer& msg, void* loc) {
  69     msg.append("  outside of Java heap\n");
  70     stringStream ss;
  71     os::print_location(&ss, (intptr_t) loc, false);
  72     msg.append("  %s\n", ss.as_string());
  73   }
  74 
  75   void print_obj_safe(ShenandoahMessageBuffer& msg, void* loc) {
  76     msg.append("  " PTR_FORMAT " - safe print, no details\n", p2i(loc));
  77     if (_heap->is_in(loc)) {
  78       ShenandoahHeapRegion* r = _heap->heap_region_containing(loc);
  79       if (r != NULL) {
  80         stringStream ss;
  81         r->print_on(&ss);
  82         msg.append("  region: %s", ss.as_string());
  83       }


 287         verify(_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
 288                "Forwardee end should be within the region");
 289       }
 290 
 291       verify(_safe_oop, obj, Metaspace::contains(fwd->klass()),
 292              "Forwardee klass pointer must go to metaspace");
 293       verify(_safe_oop, obj, obj->klass() == fwd->klass(),
 294              "Forwardee klass pointer must go to metaspace");
 295 
 296       oop fwd2 = (oop) BrooksPointer::get_raw(fwd);
 297       verify(_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2),
 298              "Double forwarding");
 299     }
 300 
 301     // ------------ obj and fwd are safe at this point --------------
 302 
 303     switch (_options._verify_marked) {
 304       case ShenandoahVerifier::_verify_marked_disable:
 305         // skip
 306         break;
 307       case ShenandoahVerifier::_verify_marked:
 308         verify(_safe_all, obj, _heap->is_marked(obj),
 309                "Must be marked");




 310         break;
 311       default:
 312         assert(false, "Unhandled mark verification");
 313     }
 314 
 315     switch (_options._verify_forwarded) {
 316       case ShenandoahVerifier::_verify_forwarded_disable:
 317         // skip
 318         break;
 319       case ShenandoahVerifier::_verify_forwarded_none: {
 320         verify(_safe_all, obj, oopDesc::unsafe_equals(obj, fwd),
 321                "Should not be forwarded");
 322         break;
 323       }
 324       case ShenandoahVerifier::_verify_forwarded_allow: {
 325         if (!oopDesc::unsafe_equals(obj, fwd)) {
 326           verify(_safe_all, obj, _heap->heap_region_containing(obj) != _heap->heap_region_containing(fwd),
 327                  "Forwardee should be in another region");
 328         }
 329         break;


 465         verify(r, !r->is_trash(),
 466                "Should not have trash regions");
 467         break;
 468       case ShenandoahVerifier::_verify_regions_nocset:
 469         verify(r, !r->is_cset(),
 470                "Should not have cset regions");
 471         break;
 472       case ShenandoahVerifier::_verify_regions_notrash_nocset:
 473         verify(r, !r->is_trash(),
 474                "Should not have trash regions");
 475         verify(r, !r->is_cset(),
 476                "Should not have cset regions");
 477         break;
 478       default:
 479         ShouldNotReachHere();
 480     }
 481 
 482     verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
 483            "Capacity should match region size");
 484 
 485     verify(r, r->bottom() <= _heap->top_at_mark_start(r->bottom()),
 486            "Region top should not be less than bottom");
 487 
 488     verify(r, _heap->top_at_mark_start(r->bottom()) <= r->top(),
 489            "TAMS should not be larger than top");
 490 
 491     verify(r, r->get_live_data_bytes() <= r->capacity(),
 492            "Live data cannot be larger than capacity");
 493 
 494     verify(r, r->garbage() <= r->capacity(),
 495            "Garbage cannot be larger than capacity");
 496 
 497     verify(r, r->used() <= r->capacity(),
 498            "Used cannot be larger than capacity");
 499 
 500     verify(r, r->get_shared_allocs() <= r->capacity(),
 501            "Shared alloc count should not be larger than capacity");
 502 
 503     verify(r, r->get_tlab_allocs() <= r->capacity(),
 504            "TLAB alloc count should not be larger than capacity");
 505 
 506     verify(r, r->get_gclab_allocs() <= r->capacity(),
 507            "GCLAB alloc count should not be larger than capacity");
 508 
 509     verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),


 616                                   _options);
 617 
 618     while (true) {
 619       size_t v = Atomic::add(1u, &_claimed) - 1;
 620       if (v < _heap->num_regions()) {
 621         ShenandoahHeapRegion* r = _regions->get(v);
 622         if (!r->is_humongous() && !r->is_trash()) {
 623           work_regular(r, stack, cl);
 624         } else if (r->is_humongous_start()) {
 625           work_humongous(r, stack, cl);
 626         }
 627       } else {
 628         break;
 629       }
 630     }
 631   }
 632 
 633   virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
 634     size_t processed = 0;
 635     HeapWord* obj = r->bottom() + BrooksPointer::word_size();
 636     if (_heap->is_marked((oop)obj)) {
 637       verify_and_follow(obj, stack, cl, &processed);
 638     }
 639     Atomic::add(processed, &_processed);
 640   }
 641 
 642   virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
 643     size_t processed = 0;
 644     MarkBitMap* mark_bit_map = _heap->mark_bit_map();
 645     HeapWord* tams = _heap->top_at_mark_start(r->bottom());
 646 
 647     // Bitmaps, before TAMS
 648     if (tams > r->bottom()) {
 649       HeapWord* start = r->bottom() + BrooksPointer::word_size();
 650       HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams);
 651 
 652       while (addr < tams) {
 653         verify_and_follow(addr, stack, cl, &processed);
 654         addr += BrooksPointer::word_size();
 655         if (addr < tams) {
 656           addr = mark_bit_map->getNextMarkedWordAddress(addr, tams);
 657         }
 658       }
 659     }
 660 
 661     // Size-based, after TAMS
 662     {
 663       HeapWord* limit = r->top();
 664       HeapWord* addr = tams + BrooksPointer::word_size();
 665 


 745   // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
 746   // This verifies what application can see, since it only cares about reachable objects.
 747   size_t count_reachable = 0;
 748   if (ShenandoahVerifyLevel >= 2) {
 749     ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(),
 750                                ShenandoahPhaseTimings::_num_phases); // no need for stats
 751 
 752     ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options);
 753     _heap->workers()->run_task(&task);
 754     count_reachable = task.processed();
 755   }
 756 
 757   // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
 758   // not the application, can see during the region scans. There is no reason to process the objects
 759   // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
 760   // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
 761   // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
 762   // version
 763 
 764   size_t count_marked = 0;
 765   if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked) {
 766     ShenandoahVerifierMarkedRegionTask task(_heap->regions(), _verification_bit_map, ld, label, options);
 767     _heap->workers()->run_task(&task);
 768     count_marked = task.processed();
 769   } else {
 770     guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_disable, "Should be");
 771   }
 772 
 773   // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
 774   // marked objects.
 775 
 776   if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked && liveness == _verify_liveness_complete) {
 777     ShenandoahHeapRegionSet* set = _heap->regions();
 778     for (size_t i = 0; i < _heap->num_regions(); i++) {
 779       ShenandoahHeapRegion* r = set->get(i);
 780 
 781       juint verf_live = 0;
 782       if (r->is_humongous()) {
 783         // For humongous objects, test if start region is marked live, and if so,
 784         // all humongous regions in that chain have live data equal to their "used".
 785         juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]);
 786         if (start_live > 0) {
 787           verf_live = (juint)(r->used() / HeapWordSize);
 788         }
 789       } else {
 790         verf_live = OrderAccess::load_acquire(&ld[r->region_number()]);
 791       }
 792 
 793       size_t reg_live = r->get_live_data_words();
 794       if (reg_live != verf_live) {
 795         ResourceMark rm;
 796         stringStream ss;


 830             _verify_liveness_disable,    // no reliable liveness data
 831             _verify_regions_notrash      // no trash regions
 832     );
 833   } else {
 834     verify_at_safepoint(
 835             "Before Mark",
 836             _verify_forwarded_none,      // UR should have fixed up
 837             _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
 838             _verify_matrix_conservative, // UR should have fixed matrix
 839             _verify_cset_none,           // UR should have fixed this
 840             _verify_liveness_disable,    // no reliable liveness data
 841             _verify_regions_notrash      // no trash regions
 842     );
 843   }
 844 }
 845 
 846 void ShenandoahVerifier::verify_after_concmark() {
 847   verify_at_safepoint(
 848           "After Mark",
 849           _verify_forwarded_none,      // no forwarded references
 850           _verify_marked,              // bitmaps as precise as we can get
 851           _verify_matrix_disable,      // matrix might be foobared
 852           _verify_cset_none,           // no references to cset anymore
 853           _verify_liveness_complete,   // liveness data must be complete here
 854           _verify_regions_disable      // trash regions not yet recycled
 855   );
 856 }
 857 
 858 void ShenandoahVerifier::verify_before_evacuation() {
 859   // Evacuation is always preceded by mark, but we want to have a sanity check after
 860   // selecting the collection set, and (immediate) regions recycling
 861   verify_at_safepoint(
 862           "Before Evacuation",
 863           _verify_forwarded_none,    // no forwarded references
 864           _verify_marked,            // walk over marked objects too
 865           _verify_matrix_disable,    // skip, verified after mark
 866           _verify_cset_disable,      // skip, verified after mark
 867           _verify_liveness_disable,  // skip, verified after mark
 868           _verify_regions_disable    // trash regions not yet recycled
 869   );
 870 }
 871 
 872 void ShenandoahVerifier::verify_after_evacuation() {
 873   verify_at_safepoint(
 874           "After Evacuation",
 875           _verify_forwarded_allow,     // objects are still forwarded
 876           _verify_marked,              // bitmaps might be stale, but alloc-after-mark should be well
 877           _verify_matrix_disable,      // matrix is inconsistent here
 878           _verify_cset_forwarded,      // all cset refs are fully forwarded
 879           _verify_liveness_disable,    // no reliable liveness data anymore
 880           _verify_regions_notrash      // trash regions have been recycled already
 881   );
 882 }
 883 
 884 void ShenandoahVerifier::verify_before_updaterefs() {
 885   verify_at_safepoint(
 886           "Before Updating References",
 887           _verify_forwarded_allow,     // forwarded references allowed
 888           _verify_marked,              // bitmaps might be stale, but alloc-after-mark should be well
 889           _verify_matrix_disable,      // matrix is inconsistent here
 890           _verify_cset_forwarded,      // all cset refs are fully forwarded
 891           _verify_liveness_disable,    // no reliable liveness data anymore
 892           _verify_regions_notrash      // trash regions have been recycled already
 893   );
 894 }
 895 
 896 void ShenandoahVerifier::verify_after_updaterefs() {
 897   verify_at_safepoint(
 898           "After Updating References",
 899           _verify_forwarded_none,      // no forwarded references
 900           _verify_marked,              // bitmaps might be stale, but alloc-after-mark should be well
 901           _verify_matrix_conservative, // matrix is conservatively consistent
 902           _verify_cset_none,           // no cset references, all updated
 903           _verify_liveness_disable,    // no reliable liveness data anymore
 904           _verify_regions_nocset       // no cset regions, trash regions have appeared
 905   );
 906 }
 907 
 908 void ShenandoahVerifier::verify_before_partial() {
 909   verify_at_safepoint(
 910           "Before Partial GC",
 911           _verify_forwarded_none,      // cannot have forwarded objects
 912           _verify_marked,              // bitmaps might be stale, but alloc-after-mark should be well
 913           _verify_matrix_conservative, // matrix is conservatively consistent
 914           _verify_cset_none,           // no cset references before partial
 915           _verify_liveness_disable,    // no reliable liveness data anymore
 916           _verify_regions_notrash_nocset // no trash and no cset regions
 917   );
 918 }
 919 
 920 void ShenandoahVerifier::verify_after_partial() {
 921   verify_at_safepoint(
 922           "After Partial GC",
 923           _verify_forwarded_none,      // cannot have forwarded objects
 924           _verify_marked,              // bitmaps might be stale, but alloc-after-mark should be well
 925           _verify_matrix_conservative, // matrix is conservatively consistent
 926           _verify_cset_none,           // no cset references left after partial
 927           _verify_liveness_disable,    // no reliable liveness data anymore
 928           _verify_regions_nocset       // no cset regions, trash regions allowed
 929   );
 930 }
 931 
 932 void ShenandoahVerifier::verify_before_fullgc() {
 933   verify_at_safepoint(
 934           "Before Full GC",
 935           _verify_forwarded_allow,     // can have forwarded objects
 936           _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
 937           _verify_matrix_disable,      // matrix might be foobared
 938           _verify_cset_disable,        // cset might be foobared
 939           _verify_liveness_disable,    // no reliable liveness data anymore
 940           _verify_regions_disable      // no reliable region data here
 941   );
 942 }
 943 
 944 void ShenandoahVerifier::verify_after_fullgc() {
 945   verify_at_safepoint(
 946           "After Full GC",
 947           _verify_forwarded_none,      // all objects are non-forwarded
 948           _verify_marked,              // all objects are marked
 949           _verify_matrix_conservative, // matrix is conservatively consistent
 950           _verify_cset_none,           // no cset references
 951           _verify_liveness_disable,    // no reliable liveness data anymore
 952           _verify_regions_notrash_nocset // no trash, no cset
 953   );
 954 }
 955 
 956 void ShenandoahVerifier::verify_oop_fwdptr(oop obj, oop fwd) {
 957   guarantee(UseShenandoahGC, "must only be called when Shenandoah is used");
 958 
 959   ShenandoahHeap* heap = ShenandoahHeap::heap();
 960 
 961   guarantee(obj != NULL, "oop is not NULL");
 962   guarantee(heap->is_in(obj), "oop must point to a heap address: " PTR_FORMAT, p2i(obj));
 963 
 964   guarantee(fwd != NULL, "forwardee is not NULL");
 965   if (!heap->is_in(fwd)) {
 966     ResourceMark rm;
 967     ShenandoahHeapRegion* r = heap->heap_region_containing(obj);
 968     stringStream obj_region;


< prev index next >