120 "Object end should be within the region");
121 } else {
122 size_t humongous_start = obj_reg->region_number();
123 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
124 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
125 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
126 "Humongous object is in continuation that fits it");
127 }
128 }
129
130 // ------------ obj is safe at this point --------------
131
132 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
133 "Object should be in active region");
134
135 switch (_options._verify_liveness) {
136 case ShenandoahVerifier::_verify_liveness_disable:
137 // skip
138 break;
139 case ShenandoahVerifier::_verify_liveness_complete:
140 Atomic::add(obj->size() + ShenandoahForwarding::word_size(), &_ld[obj_reg->region_number()]);
141 // fallthrough for fast failure for un-live regions:
142 case ShenandoahVerifier::_verify_liveness_conservative:
143 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
144 "Object must belong to region with live data");
145 break;
146 default:
147 assert(false, "Unhandled liveness verification");
148 }
149 }
150
151 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
152
153 ShenandoahHeapRegion* fwd_reg = NULL;
154
155 if (!oopDesc::equals_raw(obj, fwd)) {
156 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
157 "Forwardee must be in heap");
158 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
159 "Forwardee is set");
160 check(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd),
260 }
261
262 /**
263 * Verify object without known interior reference.
264 * Useful when picking up the object at known offset in heap,
265 * but without knowing what objects reference it.
266 * @param obj verified object
267 */
268 void verify_oop_standalone(oop obj) {
269 _interior_loc = NULL;
270 verify_oop(obj);
271 _interior_loc = NULL;
272 }
273
274 /**
275 * Verify oop fields from this object.
276 * @param obj host object for verified fields
277 */
278 void verify_oops_from(oop obj) {
279 _loc = obj;
280 obj->oop_iterate(this);
281 _loc = NULL;
282 }
283
284 virtual void do_oop(oop* p) { do_oop_work(p); }
285 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
286 };
287
288 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
289 private:
290 size_t _used, _committed, _garbage;
291 public:
292 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
293
294 void heap_region_do(ShenandoahHeapRegion* r) {
295 _used += r->used();
296 _garbage += r->garbage();
297 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
298 }
299
511 ShenandoahMessageBuffer("%s, Marked", _label),
512 _options);
513
514 while (true) {
515 size_t v = Atomic::add(1u, &_claimed) - 1;
516 if (v < _heap->num_regions()) {
517 ShenandoahHeapRegion* r = _heap->get_region(v);
518 if (!r->is_humongous() && !r->is_trash()) {
519 work_regular(r, stack, cl);
520 } else if (r->is_humongous_start()) {
521 work_humongous(r, stack, cl);
522 }
523 } else {
524 break;
525 }
526 }
527 }
528
529 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
530 size_t processed = 0;
531 HeapWord* obj = r->bottom() + ShenandoahForwarding::word_size();
532 if (_heap->complete_marking_context()->is_marked((oop)obj)) {
533 verify_and_follow(obj, stack, cl, &processed);
534 }
535 Atomic::add(processed, &_processed);
536 }
537
538 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
539 size_t processed = 0;
540 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
541 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
542
543 // Bitmaps, before TAMS
544 if (tams > r->bottom()) {
545 HeapWord* start = r->bottom() + ShenandoahForwarding::word_size();
546 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
547
548 while (addr < tams) {
549 verify_and_follow(addr, stack, cl, &processed);
550 addr += ShenandoahForwarding::word_size();
551 if (addr < tams) {
552 addr = mark_bit_map->get_next_marked_addr(addr, tams);
553 }
554 }
555 }
556
557 // Size-based, after TAMS
558 {
559 HeapWord* limit = r->top();
560 HeapWord* addr = tams + ShenandoahForwarding::word_size();
561
562 while (addr < limit) {
563 verify_and_follow(addr, stack, cl, &processed);
564 addr += oop(addr)->size() + ShenandoahForwarding::word_size();
565 }
566 }
567
568 Atomic::add(processed, &_processed);
569 }
570
571 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
572 if (!_bitmap->par_mark(addr)) return;
573
574 // Verify the object itself:
575 oop obj = oop(addr);
576 cl.verify_oop_standalone(obj);
577
578 // Verify everything reachable from that object too, hopefully realizing
579 // everything was already marked, and never touching further:
580 cl.verify_oops_from(obj);
581 (*processed)++;
582
583 while (!stack.is_empty()) {
584 ShenandoahVerifierTask task = stack.pop();
|
120 "Object end should be within the region");
121 } else {
122 size_t humongous_start = obj_reg->region_number();
123 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
124 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
125 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
126 "Humongous object is in continuation that fits it");
127 }
128 }
129
130 // ------------ obj is safe at this point --------------
131
132 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
133 "Object should be in active region");
134
135 switch (_options._verify_liveness) {
136 case ShenandoahVerifier::_verify_liveness_disable:
137 // skip
138 break;
139 case ShenandoahVerifier::_verify_liveness_complete:
140 Atomic::add((uint) obj->size(), &_ld[obj_reg->region_number()]);
141 // fallthrough for fast failure for un-live regions:
142 case ShenandoahVerifier::_verify_liveness_conservative:
143 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
144 "Object must belong to region with live data");
145 break;
146 default:
147 assert(false, "Unhandled liveness verification");
148 }
149 }
150
151 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
152
153 ShenandoahHeapRegion* fwd_reg = NULL;
154
155 if (!oopDesc::equals_raw(obj, fwd)) {
156 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
157 "Forwardee must be in heap");
158 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
159 "Forwardee is set");
160 check(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd),
260 }
261
262 /**
263 * Verify object without known interior reference.
264 * Useful when picking up the object at known offset in heap,
265 * but without knowing what objects reference it.
266 * @param obj verified object
267 */
268 void verify_oop_standalone(oop obj) {
269 _interior_loc = NULL;
270 verify_oop(obj);
271 _interior_loc = NULL;
272 }
273
274 /**
275 * Verify oop fields from this object.
276 * @param obj host object for verified fields
277 */
278 void verify_oops_from(oop obj) {
279 _loc = obj;
280 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
281 obj->oop_iterate(this);
282 _loc = NULL;
283 }
284
285 virtual void do_oop(oop* p) { do_oop_work(p); }
286 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
287 };
288
289 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
290 private:
291 size_t _used, _committed, _garbage;
292 public:
293 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
294
295 void heap_region_do(ShenandoahHeapRegion* r) {
296 _used += r->used();
297 _garbage += r->garbage();
298 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
299 }
300
512 ShenandoahMessageBuffer("%s, Marked", _label),
513 _options);
514
515 while (true) {
516 size_t v = Atomic::add(1u, &_claimed) - 1;
517 if (v < _heap->num_regions()) {
518 ShenandoahHeapRegion* r = _heap->get_region(v);
519 if (!r->is_humongous() && !r->is_trash()) {
520 work_regular(r, stack, cl);
521 } else if (r->is_humongous_start()) {
522 work_humongous(r, stack, cl);
523 }
524 } else {
525 break;
526 }
527 }
528 }
529
530 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
531 size_t processed = 0;
532 HeapWord* obj = r->bottom();
533 if (_heap->complete_marking_context()->is_marked((oop)obj)) {
534 verify_and_follow(obj, stack, cl, &processed);
535 }
536 Atomic::add(processed, &_processed);
537 }
538
539 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
540 size_t processed = 0;
541 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
542 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
543
544 // Bitmaps, before TAMS
545 if (tams > r->bottom()) {
546 HeapWord* start = r->bottom();
547 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
548
549 while (addr < tams) {
550 verify_and_follow(addr, stack, cl, &processed);
551 addr += 1;
552 if (addr < tams) {
553 addr = mark_bit_map->get_next_marked_addr(addr, tams);
554 }
555 }
556 }
557
558 // Size-based, after TAMS
559 {
560 HeapWord* limit = r->top();
561 HeapWord* addr = tams;
562
563 while (addr < limit) {
564 verify_and_follow(addr, stack, cl, &processed);
565 addr += oop(addr)->size();
566 }
567 }
568
569 Atomic::add(processed, &_processed);
570 }
571
572 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
573 if (!_bitmap->par_mark(addr)) return;
574
575 // Verify the object itself:
576 oop obj = oop(addr);
577 cl.verify_oop_standalone(obj);
578
579 // Verify everything reachable from that object too, hopefully realizing
580 // everything was already marked, and never touching further:
581 cl.verify_oops_from(obj);
582 (*processed)++;
583
584 while (!stack.is_empty()) {
585 ShenandoahVerifierTask task = stack.pop();
|