2231
2232 ////////////////////////////////////////////////////////
2233 // CMS Verification Support
2234 ////////////////////////////////////////////////////////
2235 // Following the remark phase, the following invariant
2236 // should hold -- each object in the CMS heap which is
2237 // marked in markBitMap() should be marked in the verification_mark_bm().
2238
2239 class VerifyMarkedClosure: public BitMapClosure {
2240 CMSBitMap* _marks;
2241 bool _failed;
2242
2243 public:
2244 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2245
2246 bool do_bit(size_t offset) {
2247 HeapWord* addr = _marks->offsetToHeapWord(offset);
2248 if (!_marks->isMarked(addr)) {
2249 Log(gc, verify) log;
2250 ResourceMark rm;
2251 // Unconditional write?
2252 LogStream ls(log.error());
2253 oop(addr)->print_on(&ls);
2254 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
2255 _failed = true;
2256 }
2257 return true;
2258 }
2259
2260 bool failed() { return _failed; }
2261 };
2262
2263 bool CMSCollector::verify_after_remark() {
2264 GCTraceTime(Info, gc, phases, verify) tm("Verifying CMS Marking.");
2265 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2266 static bool init = false;
2267
2268 assert(SafepointSynchronize::is_at_safepoint(),
2269 "Else mutations in object graph will make answer suspect");
2270 assert(have_cms_token(),
2271 "Else there may be mutual interference in use of "
2358 while (_restart_addr != NULL) {
2359 // Deal with stack overflow: by restarting at the indicated
2360 // address.
2361 HeapWord* ra = _restart_addr;
2362 markFromRootsClosure.reset(ra);
2363 _restart_addr = NULL;
2364 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2365 }
2366 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2367 verify_work_stacks_empty();
2368
2369 // Marking completed -- now verify that each bit marked in
2370 // verification_mark_bm() is also marked in markBitMap(); flag all
2371 // errors by printing corresponding objects.
2372 VerifyMarkedClosure vcl(markBitMap());
2373 verification_mark_bm()->iterate(&vcl);
2374 if (vcl.failed()) {
2375 Log(gc, verify) log;
2376 log.error("Failed marking verification after remark");
2377 ResourceMark rm;
2378 // Unconditional write?
2379 LogStream ls(log.error());
2380 gch->print_on(&ls);
2381 fatal("CMS: failed marking verification after remark");
2382 }
2383 }
2384
2385 class VerifyKlassOopsKlassClosure : public KlassClosure {
2386 class VerifyKlassOopsClosure : public OopClosure {
2387 CMSBitMap* _bitmap;
2388 public:
2389 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2390 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2391 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2392 } _oop_closure;
2393 public:
2394 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2395 void do_klass(Klass* k) {
2396 k->oops_do(&_oop_closure);
2397 }
2398 };
5861 // A variant of the above, used for CMS marking verification.
5862 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
5863 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
5864 _span(span),
5865 _verification_bm(verification_bm),
5866 _cms_bm(cms_bm)
5867 {
5868 assert(ref_processor() == NULL, "deliberately left NULL");
5869 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
5870 }
5871
5872 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
5873 // if p points into _span, then mark corresponding bit in _markBitMap
5874 assert(obj->is_oop(), "expected an oop");
5875 HeapWord* addr = (HeapWord*)obj;
5876 if (_span.contains(addr)) {
5877 _verification_bm->mark(addr);
5878 if (!_cms_bm->isMarked(addr)) {
5879 Log(gc, verify) log;
5880 ResourceMark rm;
5881 // Unconditional write?
5882 LogStream ls(log.error());
5883 oop(addr)->print_on(&ls);
5884 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
5885 fatal("... aborting");
5886 }
5887 }
5888 }
5889
5890 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5891 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5892
5893 //////////////////////////////////////////////////
5894 // MarkRefsIntoAndScanClosure
5895 //////////////////////////////////////////////////
5896
5897 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
5898 ReferenceProcessor* rp,
5899 CMSBitMap* bit_map,
5900 CMSBitMap* mod_union_table,
5901 CMSMarkStack* mark_stack,
6642 // Upon stack overflow, we discard (part of) the stack,
6643 // remembering the least address amongst those discarded
6644 // in CMSCollector's _restart_address.
6645 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6646 // Remember the least grey address discarded
6647 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6648 _collector->lower_restart_addr(ra);
6649 _mark_stack->reset(); // discard stack contents
6650 _mark_stack->expand(); // expand the stack if possible
6651 }
6652
6653 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6654 assert(obj->is_oop_or_null(), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6655 HeapWord* addr = (HeapWord*)obj;
6656 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6657 // Oop lies in _span and isn't yet grey or black
6658 _verification_bm->mark(addr); // now grey
6659 if (!_cms_bm->isMarked(addr)) {
6660 Log(gc, verify) log;
6661 ResourceMark rm;
6662 // Unconditional write?
6663 LogStream ls(log.error());
6664 oop(addr)->print_on(&ls);
6665 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6666 fatal("... aborting");
6667 }
6668
6669 if (!_mark_stack->push(obj)) { // stack overflow
6670 log_trace(gc)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _mark_stack->capacity());
6671 assert(_mark_stack->isFull(), "Else push should have succeeded");
6672 handle_stack_overflow(addr);
6673 }
6674 // anything including and to the right of _finger
6675 // will be scanned as we iterate over the remainder of the
6676 // bit map
6677 }
6678 }
6679
6680 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6681 MemRegion span,
6682 CMSBitMap* bitMap, CMSMarkStack* markStack,
7051 st->print_cr("_limit = " PTR_FORMAT, p2i(_limit));
7052 st->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger));
7053 NOT_PRODUCT(st->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));)
7054 st->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d",
7055 _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced);
7056 }
7057
7058 #ifndef PRODUCT
7059 // Assertion checking only: no useful work in product mode --
7060 // however, if any of the flags below become product flags,
7061 // you may need to review this code to see if it needs to be
7062 // enabled in product mode.
7063 SweepClosure::~SweepClosure() {
7064 assert_lock_strong(_freelistLock);
7065 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7066 "sweep _limit out of bounds");
7067 if (inFreeRange()) {
7068 Log(gc, sweep) log;
7069 log.error("inFreeRange() should have been reset; dumping state of SweepClosure");
7070 ResourceMark rm;
7071 // Unconditional write?
7072 LogStream ls(log.error());
7073 print_on(&ls);
7074 ShouldNotReachHere();
7075 }
7076
7077 if (log_is_enabled(Debug, gc, sweep)) {
7078 log_debug(gc, sweep)("Collected " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7079 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7080 log_debug(gc, sweep)("Live " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes Already free " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7081 _numObjectsLive, _numWordsLive*sizeof(HeapWord), _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7082 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) * sizeof(HeapWord);
7083 log_debug(gc, sweep)("Total sweep: " SIZE_FORMAT " bytes", totalBytes);
7084 }
7085
7086 if (log_is_enabled(Trace, gc, sweep) && CMSVerifyReturnedBytes) {
7087 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7088 size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes();
7089 size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes;
7090 log_trace(gc, sweep)("Returned " SIZE_FORMAT " bytes Indexed List Returned " SIZE_FORMAT " bytes Dictionary Returned " SIZE_FORMAT " bytes",
7091 returned_bytes, indexListReturnedBytes, dict_returned_bytes);
|
2231
2232 ////////////////////////////////////////////////////////
2233 // CMS Verification Support
2234 ////////////////////////////////////////////////////////
2235 // Following the remark phase, the following invariant
2236 // should hold -- each object in the CMS heap which is
2237 // marked in markBitMap() should be marked in the verification_mark_bm().
2238
2239 class VerifyMarkedClosure: public BitMapClosure {
2240 CMSBitMap* _marks;
2241 bool _failed;
2242
2243 public:
2244 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2245
2246 bool do_bit(size_t offset) {
2247 HeapWord* addr = _marks->offsetToHeapWord(offset);
2248 if (!_marks->isMarked(addr)) {
2249 Log(gc, verify) log;
2250 ResourceMark rm;
2251 LogStream ls(log.error());
2252 oop(addr)->print_on(&ls);
2253 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
2254 _failed = true;
2255 }
2256 return true;
2257 }
2258
2259 bool failed() { return _failed; }
2260 };
2261
2262 bool CMSCollector::verify_after_remark() {
2263 GCTraceTime(Info, gc, phases, verify) tm("Verifying CMS Marking.");
2264 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2265 static bool init = false;
2266
2267 assert(SafepointSynchronize::is_at_safepoint(),
2268 "Else mutations in object graph will make answer suspect");
2269 assert(have_cms_token(),
2270 "Else there may be mutual interference in use of "
2357 while (_restart_addr != NULL) {
2358 // Deal with stack overflow: by restarting at the indicated
2359 // address.
2360 HeapWord* ra = _restart_addr;
2361 markFromRootsClosure.reset(ra);
2362 _restart_addr = NULL;
2363 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2364 }
2365 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2366 verify_work_stacks_empty();
2367
2368 // Marking completed -- now verify that each bit marked in
2369 // verification_mark_bm() is also marked in markBitMap(); flag all
2370 // errors by printing corresponding objects.
2371 VerifyMarkedClosure vcl(markBitMap());
2372 verification_mark_bm()->iterate(&vcl);
2373 if (vcl.failed()) {
2374 Log(gc, verify) log;
2375 log.error("Failed marking verification after remark");
2376 ResourceMark rm;
2377 LogStream ls(log.error());
2378 gch->print_on(&ls);
2379 fatal("CMS: failed marking verification after remark");
2380 }
2381 }
2382
2383 class VerifyKlassOopsKlassClosure : public KlassClosure {
2384 class VerifyKlassOopsClosure : public OopClosure {
2385 CMSBitMap* _bitmap;
2386 public:
2387 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2388 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2389 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2390 } _oop_closure;
2391 public:
2392 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2393 void do_klass(Klass* k) {
2394 k->oops_do(&_oop_closure);
2395 }
2396 };
5859 // A variant of the above, used for CMS marking verification.
5860 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
5861 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
5862 _span(span),
5863 _verification_bm(verification_bm),
5864 _cms_bm(cms_bm)
5865 {
5866 assert(ref_processor() == NULL, "deliberately left NULL");
5867 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
5868 }
5869
5870 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
5871 // if p points into _span, then mark corresponding bit in _markBitMap
5872 assert(obj->is_oop(), "expected an oop");
5873 HeapWord* addr = (HeapWord*)obj;
5874 if (_span.contains(addr)) {
5875 _verification_bm->mark(addr);
5876 if (!_cms_bm->isMarked(addr)) {
5877 Log(gc, verify) log;
5878 ResourceMark rm;
5879 LogStream ls(log.error());
5880 oop(addr)->print_on(&ls);
5881 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
5882 fatal("... aborting");
5883 }
5884 }
5885 }
5886
5887 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5888 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5889
5890 //////////////////////////////////////////////////
5891 // MarkRefsIntoAndScanClosure
5892 //////////////////////////////////////////////////
5893
5894 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
5895 ReferenceProcessor* rp,
5896 CMSBitMap* bit_map,
5897 CMSBitMap* mod_union_table,
5898 CMSMarkStack* mark_stack,
6639 // Upon stack overflow, we discard (part of) the stack,
6640 // remembering the least address amongst those discarded
6641 // in CMSCollector's _restart_address.
6642 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6643 // Remember the least grey address discarded
6644 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6645 _collector->lower_restart_addr(ra);
6646 _mark_stack->reset(); // discard stack contents
6647 _mark_stack->expand(); // expand the stack if possible
6648 }
6649
6650 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6651 assert(obj->is_oop_or_null(), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6652 HeapWord* addr = (HeapWord*)obj;
6653 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6654 // Oop lies in _span and isn't yet grey or black
6655 _verification_bm->mark(addr); // now grey
6656 if (!_cms_bm->isMarked(addr)) {
6657 Log(gc, verify) log;
6658 ResourceMark rm;
6659 LogStream ls(log.error());
6660 oop(addr)->print_on(&ls);
6661 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6662 fatal("... aborting");
6663 }
6664
6665 if (!_mark_stack->push(obj)) { // stack overflow
6666 log_trace(gc)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _mark_stack->capacity());
6667 assert(_mark_stack->isFull(), "Else push should have succeeded");
6668 handle_stack_overflow(addr);
6669 }
6670 // anything including and to the right of _finger
6671 // will be scanned as we iterate over the remainder of the
6672 // bit map
6673 }
6674 }
6675
6676 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6677 MemRegion span,
6678 CMSBitMap* bitMap, CMSMarkStack* markStack,
7047 st->print_cr("_limit = " PTR_FORMAT, p2i(_limit));
7048 st->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger));
7049 NOT_PRODUCT(st->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));)
7050 st->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d",
7051 _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced);
7052 }
7053
7054 #ifndef PRODUCT
7055 // Assertion checking only: no useful work in product mode --
7056 // however, if any of the flags below become product flags,
7057 // you may need to review this code to see if it needs to be
7058 // enabled in product mode.
7059 SweepClosure::~SweepClosure() {
7060 assert_lock_strong(_freelistLock);
7061 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7062 "sweep _limit out of bounds");
7063 if (inFreeRange()) {
7064 Log(gc, sweep) log;
7065 log.error("inFreeRange() should have been reset; dumping state of SweepClosure");
7066 ResourceMark rm;
7067 LogStream ls(log.error());
7068 print_on(&ls);
7069 ShouldNotReachHere();
7070 }
7071
7072 if (log_is_enabled(Debug, gc, sweep)) {
7073 log_debug(gc, sweep)("Collected " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7074 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7075 log_debug(gc, sweep)("Live " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes Already free " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7076 _numObjectsLive, _numWordsLive*sizeof(HeapWord), _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7077 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) * sizeof(HeapWord);
7078 log_debug(gc, sweep)("Total sweep: " SIZE_FORMAT " bytes", totalBytes);
7079 }
7080
7081 if (log_is_enabled(Trace, gc, sweep) && CMSVerifyReturnedBytes) {
7082 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7083 size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes();
7084 size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes;
7085 log_trace(gc, sweep)("Returned " SIZE_FORMAT " bytes Indexed List Returned " SIZE_FORMAT " bytes Dictionary Returned " SIZE_FORMAT " bytes",
7086 returned_bytes, indexListReturnedBytes, dict_returned_bytes);
|