122 "Object end should be within the region");
123 } else {
124 size_t humongous_start = obj_reg->region_number();
125 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
126 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
127 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
128 "Humongous object is in continuation that fits it");
129 }
130 }
131
132 // ------------ obj is safe at this point --------------
133
134 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
135 "Object should be in active region");
136
137 switch (_options._verify_liveness) {
138 case ShenandoahVerifier::_verify_liveness_disable:
139 // skip
140 break;
141 case ShenandoahVerifier::_verify_liveness_complete:
142 Atomic::add((uint) obj->size(), &_ld[obj_reg->region_number()]);
143 // fallthrough for fast failure for un-live regions:
144 case ShenandoahVerifier::_verify_liveness_conservative:
145 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
146 "Object must belong to region with live data");
147 break;
148 default:
149 assert(false, "Unhandled liveness verification");
150 }
151 }
152
153 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
154
155 ShenandoahHeapRegion* fwd_reg = NULL;
156
157 if (obj != fwd) {
158 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
159 "Forwardee must be in heap");
160 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
161 "Forwardee is set");
162 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
462 if (_heap->unload_classes()) {
463 _verifier->strong_roots_do(&cl);
464 } else {
465 _verifier->roots_do(&cl);
466 }
467 }
468
469 size_t processed = 0;
470
471 if (ShenandoahVerifyLevel >= 3) {
472 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
473 ShenandoahMessageBuffer("%s, Reachable", _label),
474 _options);
475 while (!stack.is_empty()) {
476 processed++;
477 ShenandoahVerifierTask task = stack.pop();
478 cl.verify_oops_from(task.obj());
479 }
480 }
481
482 Atomic::add(processed, &_processed);
483 }
484 };
485
486 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask {
487 private:
488 const char* _label;
489 ShenandoahVerifier::VerifyOptions _options;
490 ShenandoahHeap *_heap;
491 MarkBitMap* _bitmap;
492 ShenandoahLivenessData* _ld;
493 volatile size_t _claimed;
494 volatile size_t _processed;
495
496 public:
497 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
498 ShenandoahLivenessData* ld,
499 const char* label,
500 ShenandoahVerifier::VerifyOptions options) :
501 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"),
502 _label(label),
503 _options(options),
504 _heap(ShenandoahHeap::heap()),
505 _bitmap(bitmap),
506 _ld(ld),
507 _claimed(0),
508 _processed(0) {};
509
510 size_t processed() {
511 return _processed;
512 }
513
514 virtual void work(uint worker_id) {
515 ShenandoahVerifierStack stack;
516 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
517 ShenandoahMessageBuffer("%s, Marked", _label),
518 _options);
519
520 while (true) {
521 size_t v = Atomic::add(1u, &_claimed) - 1;
522 if (v < _heap->num_regions()) {
523 ShenandoahHeapRegion* r = _heap->get_region(v);
524 if (!r->is_humongous() && !r->is_trash()) {
525 work_regular(r, stack, cl);
526 } else if (r->is_humongous_start()) {
527 work_humongous(r, stack, cl);
528 }
529 } else {
530 break;
531 }
532 }
533 }
534
535 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
536 size_t processed = 0;
537 HeapWord* obj = r->bottom();
538 if (_heap->complete_marking_context()->is_marked((oop)obj)) {
539 verify_and_follow(obj, stack, cl, &processed);
540 }
541 Atomic::add(processed, &_processed);
542 }
543
544 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
545 size_t processed = 0;
546 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
547 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
548
549 // Bitmaps, before TAMS
550 if (tams > r->bottom()) {
551 HeapWord* start = r->bottom();
552 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
553
554 while (addr < tams) {
555 verify_and_follow(addr, stack, cl, &processed);
556 addr += 1;
557 if (addr < tams) {
558 addr = mark_bit_map->get_next_marked_addr(addr, tams);
559 }
560 }
561 }
562
563 // Size-based, after TAMS
564 {
565 HeapWord* limit = r->top();
566 HeapWord* addr = tams;
567
568 while (addr < limit) {
569 verify_and_follow(addr, stack, cl, &processed);
570 addr += oop(addr)->size();
571 }
572 }
573
574 Atomic::add(processed, &_processed);
575 }
576
577 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
578 if (!_bitmap->par_mark(addr)) return;
579
580 // Verify the object itself:
581 oop obj = oop(addr);
582 cl.verify_oop_standalone(obj);
583
584 // Verify everything reachable from that object too, hopefully realizing
585 // everything was already marked, and never touching further:
586 cl.verify_oops_from(obj);
587 (*processed)++;
588
589 while (!stack.is_empty()) {
590 ShenandoahVerifierTask task = stack.pop();
591 cl.verify_oops_from(task.obj());
592 (*processed)++;
593 }
594 }
|
122 "Object end should be within the region");
123 } else {
124 size_t humongous_start = obj_reg->region_number();
125 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
126 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
127 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
128 "Humongous object is in continuation that fits it");
129 }
130 }
131
132 // ------------ obj is safe at this point --------------
133
134 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
135 "Object should be in active region");
136
137 switch (_options._verify_liveness) {
138 case ShenandoahVerifier::_verify_liveness_disable:
139 // skip
140 break;
141 case ShenandoahVerifier::_verify_liveness_complete:
142 Atomic::add(&_ld[obj_reg->region_number()], (uint) obj->size());
143 // fallthrough for fast failure for un-live regions:
144 case ShenandoahVerifier::_verify_liveness_conservative:
145 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
146 "Object must belong to region with live data");
147 break;
148 default:
149 assert(false, "Unhandled liveness verification");
150 }
151 }
152
153 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
154
155 ShenandoahHeapRegion* fwd_reg = NULL;
156
157 if (obj != fwd) {
158 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
159 "Forwardee must be in heap");
160 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
161 "Forwardee is set");
162 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
462 if (_heap->unload_classes()) {
463 _verifier->strong_roots_do(&cl);
464 } else {
465 _verifier->roots_do(&cl);
466 }
467 }
468
469 size_t processed = 0;
470
471 if (ShenandoahVerifyLevel >= 3) {
472 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
473 ShenandoahMessageBuffer("%s, Reachable", _label),
474 _options);
475 while (!stack.is_empty()) {
476 processed++;
477 ShenandoahVerifierTask task = stack.pop();
478 cl.verify_oops_from(task.obj());
479 }
480 }
481
482 Atomic::add(&_processed, processed);
483 }
484 };
485
486 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask {
487 private:
488 const char* _label;
489 ShenandoahVerifier::VerifyOptions _options;
490 ShenandoahHeap *_heap;
491 MarkBitMap* _bitmap;
492 ShenandoahLivenessData* _ld;
493 volatile size_t _claimed;
494 volatile size_t _processed;
495
496 public:
497 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
498 ShenandoahLivenessData* ld,
499 const char* label,
500 ShenandoahVerifier::VerifyOptions options) :
501 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"),
502 _label(label),
503 _options(options),
504 _heap(ShenandoahHeap::heap()),
505 _bitmap(bitmap),
506 _ld(ld),
507 _claimed(0),
508 _processed(0) {};
509
510 size_t processed() {
511 return _processed;
512 }
513
514 virtual void work(uint worker_id) {
515 ShenandoahVerifierStack stack;
516 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
517 ShenandoahMessageBuffer("%s, Marked", _label),
518 _options);
519
520 while (true) {
521 size_t v = Atomic::add(&_claimed, 1u) - 1;
522 if (v < _heap->num_regions()) {
523 ShenandoahHeapRegion* r = _heap->get_region(v);
524 if (!r->is_humongous() && !r->is_trash()) {
525 work_regular(r, stack, cl);
526 } else if (r->is_humongous_start()) {
527 work_humongous(r, stack, cl);
528 }
529 } else {
530 break;
531 }
532 }
533 }
534
535 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
536 size_t processed = 0;
537 HeapWord* obj = r->bottom();
538 if (_heap->complete_marking_context()->is_marked((oop)obj)) {
539 verify_and_follow(obj, stack, cl, &processed);
540 }
541 Atomic::add(&_processed, processed);
542 }
543
544 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
545 size_t processed = 0;
546 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
547 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
548
549 // Bitmaps, before TAMS
550 if (tams > r->bottom()) {
551 HeapWord* start = r->bottom();
552 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
553
554 while (addr < tams) {
555 verify_and_follow(addr, stack, cl, &processed);
556 addr += 1;
557 if (addr < tams) {
558 addr = mark_bit_map->get_next_marked_addr(addr, tams);
559 }
560 }
561 }
562
563 // Size-based, after TAMS
564 {
565 HeapWord* limit = r->top();
566 HeapWord* addr = tams;
567
568 while (addr < limit) {
569 verify_and_follow(addr, stack, cl, &processed);
570 addr += oop(addr)->size();
571 }
572 }
573
574 Atomic::add(&_processed, processed);
575 }
576
577 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
578 if (!_bitmap->par_mark(addr)) return;
579
580 // Verify the object itself:
581 oop obj = oop(addr);
582 cl.verify_oop_standalone(obj);
583
584 // Verify everything reachable from that object too, hopefully realizing
585 // everything was already marked, and never touching further:
586 cl.verify_oops_from(obj);
587 (*processed)++;
588
589 while (!stack.is_empty()) {
590 ShenandoahVerifierTask task = stack.pop();
591 cl.verify_oops_from(task.obj());
592 (*processed)++;
593 }
594 }
|