2207
2208 ////////////////////////////////////////////////////////
2209 // CMS Verification Support
2210 ////////////////////////////////////////////////////////
2211 // Following the remark phase, the following invariant
2212 // should hold -- each object in the CMS heap which is
2213 // marked in markBitMap() should be marked in the verification_mark_bm().
2214
2215 class VerifyMarkedClosure: public BitMapClosure {
2216 CMSBitMap* _marks;
2217 bool _failed;
2218
2219 public:
2220 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2221
2222 bool do_bit(size_t offset) {
2223 HeapWord* addr = _marks->offsetToHeapWord(offset);
2224 if (!_marks->isMarked(addr)) {
2225 LogHandle(gc, verify) log;
2226 ResourceMark rm;
2227 oop(addr)->print_on(log.info_stream());
2228 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
2229 _failed = true;
2230 }
2231 return true;
2232 }
2233
2234 bool failed() { return _failed; }
2235 };
2236
2237 bool CMSCollector::verify_after_remark() {
2238 GCTraceTime(Info, gc, verify) tm("Verifying CMS Marking.");
2239 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2240 static bool init = false;
2241
2242 assert(SafepointSynchronize::is_at_safepoint(),
2243 "Else mutations in object graph will make answer suspect");
2244 assert(have_cms_token(),
2245 "Else there may be mutual interference in use of "
2246 " verification data structures");
2247 assert(_collectorState > Marking && _collectorState <= Sweeping,
2248 "Else marking info checked here may be obsolete");
2333 assert(_restart_addr == NULL, "Expected pre-condition");
2334 verification_mark_bm()->iterate(&markFromRootsClosure);
2335 while (_restart_addr != NULL) {
2336 // Deal with stack overflow: by restarting at the indicated
2337 // address.
2338 HeapWord* ra = _restart_addr;
2339 markFromRootsClosure.reset(ra);
2340 _restart_addr = NULL;
2341 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2342 }
2343 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2344 verify_work_stacks_empty();
2345
2346 // Marking completed -- now verify that each bit marked in
2347 // verification_mark_bm() is also marked in markBitMap(); flag all
2348 // errors by printing corresponding objects.
2349 VerifyMarkedClosure vcl(markBitMap());
2350 verification_mark_bm()->iterate(&vcl);
2351 if (vcl.failed()) {
2352 LogHandle(gc, verify) log;
2353 log.info("Verification failed");
2354 ResourceMark rm;
2355 gch->print_on(log.info_stream());
2356 fatal("CMS: failed marking verification after remark");
2357 }
2358 }
2359
2360 class VerifyKlassOopsKlassClosure : public KlassClosure {
2361 class VerifyKlassOopsClosure : public OopClosure {
2362 CMSBitMap* _bitmap;
2363 public:
2364 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2365 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2366 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2367 } _oop_closure;
2368 public:
2369 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2370 void do_klass(Klass* k) {
2371 k->oops_do(&_oop_closure);
2372 }
2373 };
2374
2375 void CMSCollector::verify_after_remark_work_2() {
5863 // A variant of the above, used for CMS marking verification.
5864 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
5865 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
5866 _span(span),
5867 _verification_bm(verification_bm),
5868 _cms_bm(cms_bm)
5869 {
5870 assert(ref_processor() == NULL, "deliberately left NULL");
5871 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
5872 }
5873
5874 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
5875 // if p points into _span, then mark corresponding bit in _markBitMap
5876 assert(obj->is_oop(), "expected an oop");
5877 HeapWord* addr = (HeapWord*)obj;
5878 if (_span.contains(addr)) {
5879 _verification_bm->mark(addr);
5880 if (!_cms_bm->isMarked(addr)) {
5881 LogHandle(gc, verify) log;
5882 ResourceMark rm;
5883 oop(addr)->print_on(log.info_stream());
5884 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
5885 fatal("... aborting");
5886 }
5887 }
5888 }
5889
5890 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5891 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5892
5893 //////////////////////////////////////////////////
5894 // MarkRefsIntoAndScanClosure
5895 //////////////////////////////////////////////////
5896
5897 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
5898 ReferenceProcessor* rp,
5899 CMSBitMap* bit_map,
5900 CMSBitMap* mod_union_table,
5901 CMSMarkStack* mark_stack,
5902 CMSCollector* collector,
5903 bool should_yield,
5904 bool concurrent_precleaning):
6644 // Upon stack overflow, we discard (part of) the stack,
6645 // remembering the least address amongst those discarded
6646 // in CMSCollector's _restart_address.
6647 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6648 // Remember the least grey address discarded
6649 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6650 _collector->lower_restart_addr(ra);
6651 _mark_stack->reset(); // discard stack contents
6652 _mark_stack->expand(); // expand the stack if possible
6653 }
6654
6655 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6656 assert(obj->is_oop_or_null(), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6657 HeapWord* addr = (HeapWord*)obj;
6658 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6659 // Oop lies in _span and isn't yet grey or black
6660 _verification_bm->mark(addr); // now grey
6661 if (!_cms_bm->isMarked(addr)) {
6662 LogHandle(gc, verify) log;
6663 ResourceMark rm;
6664 oop(addr)->print_on(log.info_stream());
6665 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6666 fatal("... aborting");
6667 }
6668
6669 if (!_mark_stack->push(obj)) { // stack overflow
6670 log_trace(gc)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _mark_stack->capacity());
6671 assert(_mark_stack->isFull(), "Else push should have succeeded");
6672 handle_stack_overflow(addr);
6673 }
6674 // anything including and to the right of _finger
6675 // will be scanned as we iterate over the remainder of the
6676 // bit map
6677 }
6678 }
6679
6680 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6681 MemRegion span,
6682 CMSBitMap* bitMap, CMSMarkStack* markStack,
6683 HeapWord* finger, MarkFromRootsClosure* parent) :
6684 MetadataAwareOopClosure(collector->ref_processor()),
6685 _collector(collector),
|
2207
2208 ////////////////////////////////////////////////////////
2209 // CMS Verification Support
2210 ////////////////////////////////////////////////////////
2211 // Following the remark phase, the following invariant
2212 // should hold -- each object in the CMS heap which is
2213 // marked in markBitMap() should be marked in the verification_mark_bm().
2214
2215 class VerifyMarkedClosure: public BitMapClosure {
2216 CMSBitMap* _marks;
2217 bool _failed;
2218
2219 public:
2220 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2221
2222 bool do_bit(size_t offset) {
2223 HeapWord* addr = _marks->offsetToHeapWord(offset);
2224 if (!_marks->isMarked(addr)) {
2225 LogHandle(gc, verify) log;
2226 ResourceMark rm;
2227 oop(addr)->print_on(log.error_stream());
2228 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
2229 _failed = true;
2230 }
2231 return true;
2232 }
2233
2234 bool failed() { return _failed; }
2235 };
2236
2237 bool CMSCollector::verify_after_remark() {
2238 GCTraceTime(Info, gc, verify) tm("Verifying CMS Marking.");
2239 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2240 static bool init = false;
2241
2242 assert(SafepointSynchronize::is_at_safepoint(),
2243 "Else mutations in object graph will make answer suspect");
2244 assert(have_cms_token(),
2245 "Else there may be mutual interference in use of "
2246 " verification data structures");
2247 assert(_collectorState > Marking && _collectorState <= Sweeping,
2248 "Else marking info checked here may be obsolete");
2333 assert(_restart_addr == NULL, "Expected pre-condition");
2334 verification_mark_bm()->iterate(&markFromRootsClosure);
2335 while (_restart_addr != NULL) {
2336 // Deal with stack overflow: by restarting at the indicated
2337 // address.
2338 HeapWord* ra = _restart_addr;
2339 markFromRootsClosure.reset(ra);
2340 _restart_addr = NULL;
2341 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2342 }
2343 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2344 verify_work_stacks_empty();
2345
2346 // Marking completed -- now verify that each bit marked in
2347 // verification_mark_bm() is also marked in markBitMap(); flag all
2348 // errors by printing corresponding objects.
2349 VerifyMarkedClosure vcl(markBitMap());
2350 verification_mark_bm()->iterate(&vcl);
2351 if (vcl.failed()) {
2352 LogHandle(gc, verify) log;
2353 log.error("Failed marking verification after remark");
2354 ResourceMark rm;
2355 gch->print_on(log.error_stream());
2356 fatal("CMS: failed marking verification after remark");
2357 }
2358 }
2359
2360 class VerifyKlassOopsKlassClosure : public KlassClosure {
2361 class VerifyKlassOopsClosure : public OopClosure {
2362 CMSBitMap* _bitmap;
2363 public:
2364 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2365 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2366 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2367 } _oop_closure;
2368 public:
2369 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2370 void do_klass(Klass* k) {
2371 k->oops_do(&_oop_closure);
2372 }
2373 };
2374
2375 void CMSCollector::verify_after_remark_work_2() {
5863 // A variant of the above, used for CMS marking verification.
5864 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
5865 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
5866 _span(span),
5867 _verification_bm(verification_bm),
5868 _cms_bm(cms_bm)
5869 {
5870 assert(ref_processor() == NULL, "deliberately left NULL");
5871 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
5872 }
5873
5874 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
5875 // if p points into _span, then mark corresponding bit in _markBitMap
5876 assert(obj->is_oop(), "expected an oop");
5877 HeapWord* addr = (HeapWord*)obj;
5878 if (_span.contains(addr)) {
5879 _verification_bm->mark(addr);
5880 if (!_cms_bm->isMarked(addr)) {
5881 LogHandle(gc, verify) log;
5882 ResourceMark rm;
5883 oop(addr)->print_on(log.error_stream());
5884 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
5885 fatal("... aborting");
5886 }
5887 }
5888 }
5889
5890 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5891 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5892
5893 //////////////////////////////////////////////////
5894 // MarkRefsIntoAndScanClosure
5895 //////////////////////////////////////////////////
5896
5897 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
5898 ReferenceProcessor* rp,
5899 CMSBitMap* bit_map,
5900 CMSBitMap* mod_union_table,
5901 CMSMarkStack* mark_stack,
5902 CMSCollector* collector,
5903 bool should_yield,
5904 bool concurrent_precleaning):
6644 // Upon stack overflow, we discard (part of) the stack,
6645 // remembering the least address amongst those discarded
6646 // in CMSCollector's _restart_address.
6647 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6648 // Remember the least grey address discarded
6649 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6650 _collector->lower_restart_addr(ra);
6651 _mark_stack->reset(); // discard stack contents
6652 _mark_stack->expand(); // expand the stack if possible
6653 }
6654
6655 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6656 assert(obj->is_oop_or_null(), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6657 HeapWord* addr = (HeapWord*)obj;
6658 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6659 // Oop lies in _span and isn't yet grey or black
6660 _verification_bm->mark(addr); // now grey
6661 if (!_cms_bm->isMarked(addr)) {
6662 LogHandle(gc, verify) log;
6663 ResourceMark rm;
6664 oop(addr)->print_on(log.error_stream());
6665 log.error(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6666 fatal("... aborting");
6667 }
6668
6669 if (!_mark_stack->push(obj)) { // stack overflow
6670 log_trace(gc)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _mark_stack->capacity());
6671 assert(_mark_stack->isFull(), "Else push should have succeeded");
6672 handle_stack_overflow(addr);
6673 }
6674 // anything including and to the right of _finger
6675 // will be scanned as we iterate over the remainder of the
6676 // bit map
6677 }
6678 }
6679
6680 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6681 MemRegion span,
6682 CMSBitMap* bitMap, CMSMarkStack* markStack,
6683 HeapWord* finger, MarkFromRootsClosure* parent) :
6684 MetadataAwareOopClosure(collector->ref_processor()),
6685 _collector(collector),
|