1 /*
2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "memory/allocation.hpp"
27 #include "gc_implementation/shenandoah/shenandoahAsserts.hpp"
28 #include "gc_implementation/shenandoah/shenandoahHeap.hpp"
29 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc_implementation/shenandoah/shenandoahVerifier.hpp"
31 #include "gc_implementation/shenandoah/brooksPointer.hpp"
32 #include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp"
33
34 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
35 #ifdef verify_oop
36 #undef verify_oop
37 #endif
38
39 class ShenandoahVerifyOopClosure : public ExtendedOopClosure {
40 private:
41 const char* _phase;
42 ShenandoahVerifier::VerifyOptions _options;
43 ShenandoahVerifierStack* _stack;
44 ShenandoahHeap* _heap;
45 MarkBitMap* _map;
46 ShenandoahLivenessData* _ld;
47 void* _interior_loc;
48 oop _loc;
49
50 public:
51 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
52 const char* phase, ShenandoahVerifier::VerifyOptions options) :
53 _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL),
54 _phase(phase), _options(options) {};
55
56 private:
57 void verify(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
58 if (!test) {
59 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
60 }
61 }
62
63 template <class T>
64 void do_oop_work(T* p) {
65 T o = oopDesc::load_heap_oop(p);
66 if (!oopDesc::is_null(o)) {
67 oop obj = oopDesc::decode_heap_oop_not_null(o);
68
69 // Single threaded verification can use faster non-atomic stack and bitmap
70 // methods.
71 //
72 // For performance reasons, only fully verify non-marked field values.
73 // We are here when the host object for *p is already marked.
74
75 HeapWord* addr = (HeapWord*) obj;
76 if (_map->parMark(addr)) {
77 verify_oop_at(p, obj);
78 _stack->push(ShenandoahVerifierTask(obj));
79 }
80 }
81 }
82
83 void verify_oop(oop obj) {
84 // Perform consistency checks with gradually decreasing safety level. This guarantees
85 // that failure report would not try to touch something that was not yet verified to be
86 // safe to process.
87
88 verify(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
89 "oop must be in heap");
90 verify(ShenandoahAsserts::_safe_unknown, obj, check_obj_alignment(obj),
91 "oop must be aligned");
92
93 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
94 Klass* obj_klass = obj->klass_or_null();
95
96 // Verify that obj is not in dead space:
97 {
98 // Do this before touching obj->size()
99 verify(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL,
100 "Object klass pointer should not be NULL");
101 verify(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
102 "Object klass pointer must go to metaspace");
103
104 HeapWord *obj_addr = (HeapWord *) obj;
105 verify(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
106 "Object start should be within the region");
107
108 if (!obj_reg->is_humongous()) {
109 verify(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
110 "Object end should be within the region");
111 } else {
112 size_t humongous_start = obj_reg->region_number();
113 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
114 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
115 verify(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
116 "Humongous object is in continuation that fits it");
117 }
118 }
119
120 // ------------ obj is safe at this point --------------
121
122 verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
123 "Object should be in active region");
124
125 switch (_options._verify_liveness) {
126 case ShenandoahVerifier::_verify_liveness_disable:
127 // skip
128 break;
129 case ShenandoahVerifier::_verify_liveness_complete:
130 Atomic::add(obj->size() + BrooksPointer::word_size(), &_ld[obj_reg->region_number()]);
131 // fallthrough for fast failure for un-live regions:
132 case ShenandoahVerifier::_verify_liveness_conservative:
133 verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
134 "Object must belong to region with live data");
135 break;
136 default:
137 assert(false, "Unhandled liveness verification");
138 }
139 }
140
141 oop fwd = (oop) BrooksPointer::get_raw_unchecked(obj);
142
143 ShenandoahHeapRegion* fwd_reg = NULL;
144
145 if (!oopDesc::unsafe_equals(obj, fwd)) {
146 verify(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
147 "Forwardee must be in heap");
148 verify(ShenandoahAsserts::_safe_oop, obj, !oopDesc::is_null(fwd),
149 "Forwardee is set");
150 verify(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd),
151 "Forwardee must be aligned");
152
153 // Do this before touching fwd->size()
154 Klass* fwd_klass = fwd->klass_or_null();
155 verify(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL,
156 "Forwardee klass pointer should not be NULL");
157 verify(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
158 "Forwardee klass pointer must go to metaspace");
159 verify(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
160 "Forwardee klass pointer must go to metaspace");
161
162 fwd_reg = _heap->heap_region_containing(fwd);
163
164 // Verify that forwardee is not in the dead space:
165 verify(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
166 "Should have no humongous forwardees");
167
168 HeapWord *fwd_addr = (HeapWord *) fwd;
169 verify(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
170 "Forwardee start should be within the region");
171 verify(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
172 "Forwardee end should be within the region");
173
174 oop fwd2 = (oop) BrooksPointer::get_raw_unchecked(fwd);
175 verify(ShenandoahAsserts::_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2),
176 "Double forwarding");
177 } else {
178 fwd_reg = obj_reg;
179 }
180
181 // ------------ obj and fwd are safe at this point --------------
182
183 switch (_options._verify_marked) {
184 case ShenandoahVerifier::_verify_marked_disable:
185 // skip
186 break;
187 case ShenandoahVerifier::_verify_marked_next:
188 verify(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_next(obj),
189 "Must be marked in next bitmap");
190 break;
191 case ShenandoahVerifier::_verify_marked_complete:
192 verify(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_complete(obj),
193 "Must be marked in complete bitmap");
194 break;
195 default:
196 assert(false, "Unhandled mark verification");
197 }
198
199 switch (_options._verify_forwarded) {
200 case ShenandoahVerifier::_verify_forwarded_disable:
201 // skip
202 break;
203 case ShenandoahVerifier::_verify_forwarded_none: {
204 verify(ShenandoahAsserts::_safe_all, obj, oopDesc::unsafe_equals(obj, fwd),
205 "Should not be forwarded");
206 break;
207 }
208 case ShenandoahVerifier::_verify_forwarded_allow: {
209 if (!oopDesc::unsafe_equals(obj, fwd)) {
210 verify(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg,
211 "Forwardee should be in another region");
212 }
213 break;
214 }
215 default:
216 assert(false, "Unhandled forwarding verification");
217 }
218
219 switch (_options._verify_cset) {
220 case ShenandoahVerifier::_verify_cset_disable:
221 // skip
222 break;
223 case ShenandoahVerifier::_verify_cset_none:
224 verify(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj),
225 "Should not have references to collection set");
226 break;
227 case ShenandoahVerifier::_verify_cset_forwarded:
228 if (_heap->in_collection_set(obj)) {
229 verify(ShenandoahAsserts::_safe_all, obj, !oopDesc::unsafe_equals(obj, fwd),
230 "Object in collection set, should have forwardee");
231 }
232 break;
233 default:
234 assert(false, "Unhandled cset verification");
235 }
236 }
237
238 public:
239
240 /**
241 * Verify object with known interior reference.
242 * @param p interior reference where the object is referenced from; can be off-heap
243 * @param obj verified object
244 */
245 template <class T>
246 void verify_oop_at(T* p, oop obj) {
247 _interior_loc = p;
248 verify_oop(obj);
249 _interior_loc = NULL;
250 }
251
252 /**
253 * Verify object without known interior reference.
254 * Useful when picking up the object at known offset in heap,
255 * but without knowing what objects reference it.
256 * @param obj verified object
257 */
258 void verify_oop_standalone(oop obj) {
259 _interior_loc = NULL;
260 verify_oop(obj);
261 _interior_loc = NULL;
262 }
263
264 /**
265 * Verify oop fields from this object.
266 * @param obj host object for verified fields
267 */
268 void verify_oops_from(oop obj) {
269 _loc = obj;
270 obj->oop_iterate(this);
271 _loc = NULL;
272 }
273
274
275 void do_oop(oop* p) { do_oop_work(p); }
276 void do_oop(narrowOop* p) { do_oop_work(p); }
277 };
278
279 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
280 private:
281 size_t _used, _committed, _garbage;
282 public:
283 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
284
285 bool heap_region_do(ShenandoahHeapRegion* r) {
286 _used += r->used();
287 _garbage += r->garbage();
288 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
289 return false;
290 }
291
292 size_t used() { return _used; }
293 size_t committed() { return _committed; }
294 size_t garbage() { return _garbage; }
295 };
296
297 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure {
298 private:
299 ShenandoahHeap* _heap;
300 const char* _phase;
301 ShenandoahVerifier::VerifyRegions _regions;
302 public:
303 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) :
304 _heap(ShenandoahHeap::heap()), _regions(regions), _phase(phase) {};
305
306 void print_failure(ShenandoahHeapRegion* r, const char* label) {
307 ResourceMark rm;
308
309 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label);
310
311 stringStream ss;
312 r->print_on(&ss);
313 msg.append("%s", ss.as_string());
314
315 report_vm_error(__FILE__, __LINE__, msg.buffer());
316 }
317
318 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) {
319 if (!test) {
320 print_failure(r, msg);
321 }
322 }
323
324 bool heap_region_do(ShenandoahHeapRegion* r) {
325 switch (_regions) {
326 case ShenandoahVerifier::_verify_regions_disable:
327 break;
328 case ShenandoahVerifier::_verify_regions_notrash:
329 verify(r, !r->is_trash(),
330 "Should not have trash regions");
331 break;
332 case ShenandoahVerifier::_verify_regions_nocset:
333 verify(r, !r->is_cset(),
334 "Should not have cset regions");
335 break;
336 case ShenandoahVerifier::_verify_regions_notrash_nocset:
337 verify(r, !r->is_trash(),
338 "Should not have trash regions");
339 verify(r, !r->is_cset(),
340 "Should not have cset regions");
341 break;
342 default:
343 ShouldNotReachHere();
344 }
345
346 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
347 "Capacity should match region size");
348
349 verify(r, r->bottom() <= _heap->complete_top_at_mark_start(r->bottom()),
350 "Region top should not be less than bottom");
351
352 verify(r, _heap->complete_top_at_mark_start(r->bottom()) <= r->top(),
353 "Complete TAMS should not be larger than top");
354
355 verify(r, r->get_live_data_bytes() <= r->capacity(),
356 "Live data cannot be larger than capacity");
357
358 verify(r, r->garbage() <= r->capacity(),
359 "Garbage cannot be larger than capacity");
360
361 verify(r, r->used() <= r->capacity(),
362 "Used cannot be larger than capacity");
363
364 verify(r, r->get_shared_allocs() <= r->capacity(),
365 "Shared alloc count should not be larger than capacity");
366
367 verify(r, r->get_tlab_allocs() <= r->capacity(),
368 "TLAB alloc count should not be larger than capacity");
369
370 verify(r, r->get_gclab_allocs() <= r->capacity(),
371 "GCLAB alloc count should not be larger than capacity");
372
373 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
374 "Accurate accounting: shared + TLAB + GCLAB = used");
375
376 verify(r, !r->is_empty() || !r->has_live(),
377 "Empty regions should not have live data");
378
379 verify(r, r->is_cset() == r->in_collection_set(),
380 "Transitional: region flags and collection set agree");
381
382 return false;
383 }
384 };
385
386 class ShenandoahVerifierReachableTask : public AbstractGangTask {
387 private:
388 const char* _label;
389 ShenandoahRootProcessor* _rp;
390 ShenandoahVerifier::VerifyOptions _options;
391 ShenandoahHeap* _heap;
392 ShenandoahLivenessData* _ld;
393 MarkBitMap* _bitmap;
394 volatile jlong _processed;
395
396 public:
397 ShenandoahVerifierReachableTask(MarkBitMap* bitmap,
398 ShenandoahLivenessData* ld,
399 ShenandoahRootProcessor* rp,
400 const char* label,
401 ShenandoahVerifier::VerifyOptions options) :
402 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"),
403 _heap(ShenandoahHeap::heap()), _rp(rp), _ld(ld), _bitmap(bitmap), _processed(0),
404 _label(label), _options(options) {};
405
406 size_t processed() {
407 return (size_t) _processed;
408 }
409
410 virtual void work(uint worker_id) {
411 ResourceMark rm;
412 ShenandoahVerifierStack stack;
413
414 // On level 2, we need to only check the roots once.
415 // On level 3, we want to check the roots, and seed the local stack.
416 // It is a lesser evil to accept multiple root scans at level 3, because
417 // extended parallelism would buy us out.
418 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0))
419 || (ShenandoahVerifyLevel >= 3)) {
420 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
421 ShenandoahMessageBuffer("Shenandoah verification failed; %s, Roots", _label),
422 _options);
423 _rp->process_all_roots_slow(&cl);
424 }
425
426 jlong processed = 0;
427
428 if (ShenandoahVerifyLevel >= 3) {
429 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
430 ShenandoahMessageBuffer("Shenandoah verification failed; %s, Reachable", _label),
431 _options);
432 while (!stack.is_empty()) {
433 processed++;
434 ShenandoahVerifierTask task = stack.pop();
435 cl.verify_oops_from(task.obj());
436 }
437 }
438
439 Atomic::add(processed, &_processed);
440 }
441 };
442
443 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask {
444 private:
445 const char* _label;
446 ShenandoahVerifier::VerifyOptions _options;
447 ShenandoahHeap *_heap;
448 ShenandoahLivenessData* _ld;
449 MarkBitMap* _bitmap;
450 volatile jint _claimed;
451 volatile jlong _processed;
452
453 public:
454 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
455 ShenandoahLivenessData* ld,
456 const char* label,
457 ShenandoahVerifier::VerifyOptions options) :
458 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"),
459 _heap(ShenandoahHeap::heap()), _ld(ld), _bitmap(bitmap), _claimed(0), _processed(0),
460 _label(label), _options(options) {};
461
462 size_t processed() {
463 return (size_t) _processed;
464 }
465
466 virtual void work(uint worker_id) {
467 ShenandoahVerifierStack stack;
468 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
469 ShenandoahMessageBuffer("Shenandoah verification failed; %s, Marked", _label),
470 _options);
471 assert((size_t)max_jint >= _heap->num_regions(), "Too many regions");
472 while (true) {
473 size_t v = (size_t) (Atomic::add(1, &_claimed) - 1);
474 if (v < _heap->num_regions()) {
475 ShenandoahHeapRegion* r = _heap->get_region(v);
476 if (!r->is_humongous() && !r->is_trash()) {
477 work_regular(r, stack, cl);
478 } else if (r->is_humongous_start()) {
479 work_humongous(r, stack, cl);
480 }
481 } else {
482 break;
483 }
484 }
485 }
486
487 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
488 jlong processed = 0;
489 HeapWord* obj = r->bottom() + BrooksPointer::word_size();
490 if (_heap->is_marked_complete((oop)obj)) {
491 verify_and_follow(obj, stack, cl, &processed);
492 }
493 Atomic::add(processed, &_processed);
494 }
495
496 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
497 jlong processed = 0;
498 MarkBitMap* mark_bit_map = _heap->complete_mark_bit_map();
499 HeapWord* tams = _heap->complete_top_at_mark_start(r->bottom());
500
501 // Bitmaps, before TAMS
502 if (tams > r->bottom()) {
503 HeapWord* start = r->bottom() + BrooksPointer::word_size();
504 HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams);
505
506 while (addr < tams) {
507 verify_and_follow(addr, stack, cl, &processed);
508 addr += BrooksPointer::word_size();
509 if (addr < tams) {
510 addr = mark_bit_map->getNextMarkedWordAddress(addr, tams);
511 }
512 }
513 }
514
515 // Size-based, after TAMS
516 {
517 HeapWord* limit = r->top();
518 HeapWord* addr = tams + BrooksPointer::word_size();
519
520 while (addr < limit) {
521 verify_and_follow(addr, stack, cl, &processed);
522 addr += oop(addr)->size() + BrooksPointer::word_size();
523 }
524 }
525
526 Atomic::add(processed, &_processed);
527 }
528
529 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, jlong *processed) {
530 if (!_bitmap->parMark(addr)) return;
531
532 // Verify the object itself:
533 oop obj = oop(addr);
534 cl.verify_oop_standalone(obj);
535
536 // Verify everything reachable from that object too, hopefully realizing
537 // everything was already marked, and never touching further:
538 cl.verify_oops_from(obj);
539 (*processed)++;
540
541 while (!stack.is_empty()) {
542 ShenandoahVerifierTask task = stack.pop();
543 cl.verify_oops_from(task.obj());
544 (*processed)++;
545 }
546 }
547 };
548
549 void ShenandoahVerifier::verify_at_safepoint(const char *label,
550 VerifyForwarded forwarded, VerifyMarked marked,
551 VerifyMatrix matrix, VerifyCollectionSet cset,
552 VerifyLiveness liveness, VerifyRegions regions) {
553 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
554 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
555
556 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
557 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
558
559 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel);
560
561 // Heap size checks
562 {
563 ShenandoahHeapLocker lock(_heap->lock());
564
565 ShenandoahCalculateRegionStatsClosure cl;
566 _heap->heap_region_iterate(&cl);
567 size_t heap_used = _heap->used();
568 guarantee(cl.used() == heap_used,
569 err_msg("%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "K, regions-used = " SIZE_FORMAT "K",
570 label, heap_used/K, cl.used()/K));
571
572 size_t heap_committed = _heap->committed();
573 guarantee(cl.committed() == heap_committed,
574 err_msg("%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "K, regions-committed = " SIZE_FORMAT "K",
575 label, heap_committed/K, cl.committed()/K));
576 }
577
578 // Internal heap region checks
579 if (ShenandoahVerifyLevel >= 1) {
580 ShenandoahVerifyHeapRegionClosure cl(label, regions);
581 _heap->heap_region_iterate(&cl,
582 /* skip_cset = */ false,
583 /* skip_humongous_cont = */ false);
584 }
585
586 OrderAccess::fence();
587 _heap->make_tlabs_parsable(false);
588
589 // Allocate temporary bitmap for storing marking wavefront:
590 MemRegion mr = MemRegion(_verification_bit_map->startWord(), _verification_bit_map->endWord());
591 _verification_bit_map->clear_range_large(mr);
592
593 // Allocate temporary array for storing liveness data
594 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC);
595 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0);
596
597 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, matrix, cset, liveness, regions);
598
599 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
600 // This verifies what application can see, since it only cares about reachable objects.
601 size_t count_reachable = 0;
602 if (ShenandoahVerifyLevel >= 2) {
603 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(),
604 ShenandoahPhaseTimings::_num_phases); // no need for stats
605
606 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options);
607 _heap->workers()->run_task(&task);
608 count_reachable = task.processed();
609 }
610
611 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
612 // not the application, can see during the region scans. There is no reason to process the objects
613 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
614 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
615 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
616 // version
617
618 size_t count_marked = 0;
619 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
620 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options);
621 _heap->workers()->run_task(&task);
622 count_marked = task.processed();
623 } else {
624 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_next || marked == _verify_marked_disable, "Should be");
625 }
626
627 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
628 // marked objects.
629
630 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
631 for (size_t i = 0; i < _heap->num_regions(); i++) {
632 ShenandoahHeapRegion* r = _heap->get_region(i);
633
634 jint verf_live = 0;
635 if (r->is_humongous()) {
636 // For humongous objects, test if start region is marked live, and if so,
637 // all humongous regions in that chain have live data equal to their "used".
638 jint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]);
639 if (start_live > 0) {
640 verf_live = (jint)(r->used() / HeapWordSize);
641 }
642 } else {
643 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]);
644 }
645
646 size_t reg_live = r->get_live_data_words();
647 if (reg_live != (size_t)verf_live) {
648 ResourceMark rm;
649 stringStream ss;
650 r->print_on(&ss);
651 fatal(err_msg("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " INT32_FORMAT "\n%s",
652 label, reg_live, verf_live, ss.as_string()));
653 }
654 }
655 }
656
657 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)",
658 label, ShenandoahVerifyLevel, count_reachable, count_marked);
659
660 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld, mtGC);
661 }
662
663 void ShenandoahVerifier::verify_generic(VerifyOption vo) {
664 verify_at_safepoint(
665 "Generic Verification",
666 _verify_forwarded_allow, // conservatively allow forwarded
667 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
668 _verify_matrix_disable, // matrix can be inconsistent here
669 _verify_cset_disable, // cset may be inconsistent
670 _verify_liveness_disable, // no reliable liveness data
671 _verify_regions_disable // no reliable region data
672 );
673 }
674
675 void ShenandoahVerifier::verify_before_concmark() {
676 if (_heap->has_forwarded_objects()) {
677 verify_at_safepoint(
678 "Before Mark",
679 _verify_forwarded_allow, // may have forwarded references
680 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
681 _verify_matrix_disable, // matrix is foobared
682 _verify_cset_forwarded, // allow forwarded references to cset
683 _verify_liveness_disable, // no reliable liveness data
684 _verify_regions_notrash // no trash regions
685 );
686 } else {
687 verify_at_safepoint(
688 "Before Mark",
689 _verify_forwarded_none, // UR should have fixed up
690 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
691 _verify_matrix_conservative, // UR should have fixed matrix
692 _verify_cset_none, // UR should have fixed this
693 _verify_liveness_disable, // no reliable liveness data
694 _verify_regions_notrash // no trash regions
695 );
696 }
697 }
698
699 void ShenandoahVerifier::verify_after_concmark() {
700 verify_at_safepoint(
701 "After Mark",
702 _verify_forwarded_none, // no forwarded references
703 _verify_marked_complete, // bitmaps as precise as we can get
704 _verify_matrix_disable, // matrix might be foobared
705 _verify_cset_none, // no references to cset anymore
706 _verify_liveness_complete, // liveness data must be complete here
707 _verify_regions_disable // trash regions not yet recycled
708 );
709 }
710
711 void ShenandoahVerifier::verify_before_evacuation() {
712 // Evacuation is always preceded by mark, but we want to have a sanity check after
713 // selecting the collection set, and (immediate) regions recycling
714 verify_at_safepoint(
715 "Before Evacuation",
716 _verify_forwarded_none, // no forwarded references
717 _verify_marked_complete, // walk over marked objects too
718 _verify_matrix_disable, // skip, verified after mark
719 _verify_cset_disable, // skip, verified after mark
720 _verify_liveness_disable, // skip, verified after mark
721 _verify_regions_disable // trash regions not yet recycled
722 );
723 }
724
725 void ShenandoahVerifier::verify_after_evacuation() {
726 verify_at_safepoint(
727 "After Evacuation",
728 _verify_forwarded_allow, // objects are still forwarded
729 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
730 _verify_matrix_disable, // matrix is inconsistent here
731 _verify_cset_forwarded, // all cset refs are fully forwarded
732 _verify_liveness_disable, // no reliable liveness data anymore
733 _verify_regions_notrash // trash regions have been recycled already
734 );
735 }
736
737 void ShenandoahVerifier::verify_before_updaterefs() {
738 verify_at_safepoint(
739 "Before Updating References",
740 _verify_forwarded_allow, // forwarded references allowed
741 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
742 _verify_matrix_disable, // matrix is inconsistent here
743 _verify_cset_forwarded, // all cset refs are fully forwarded
744 _verify_liveness_disable, // no reliable liveness data anymore
745 _verify_regions_notrash // trash regions have been recycled already
746 );
747 }
748
749 void ShenandoahVerifier::verify_after_updaterefs() {
750 verify_at_safepoint(
751 "After Updating References",
752 _verify_forwarded_none, // no forwarded references
753 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
754 _verify_matrix_conservative, // matrix is conservatively consistent
755 _verify_cset_none, // no cset references, all updated
756 _verify_liveness_disable, // no reliable liveness data anymore
757 _verify_regions_nocset // no cset regions, trash regions have appeared
758 );
759 }
760
761 void ShenandoahVerifier::verify_before_partial() {
762 verify_at_safepoint(
763 "Before Partial GC",
764 _verify_forwarded_none, // cannot have forwarded objects
765 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
766 _verify_matrix_conservative, // matrix is conservatively consistent
767 _verify_cset_none, // no cset references before partial
768 _verify_liveness_disable, // no reliable liveness data anymore
769 _verify_regions_notrash_nocset // no trash and no cset regions
770 );
771 }
772
773 void ShenandoahVerifier::verify_after_partial() {
774 verify_at_safepoint(
775 "After Partial GC",
776 _verify_forwarded_none, // cannot have forwarded objects
777 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
778 _verify_matrix_conservative, // matrix is conservatively consistent
779 _verify_cset_none, // no cset references left after partial
780 _verify_liveness_disable, // no reliable liveness data anymore
781 _verify_regions_nocset // no cset regions, trash regions allowed
782 );
783 }
784
785 void ShenandoahVerifier::verify_before_fullgc() {
786 verify_at_safepoint(
787 "Before Full GC",
788 _verify_forwarded_allow, // can have forwarded objects
789 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
790 _verify_matrix_disable, // matrix might be foobared
791 _verify_cset_disable, // cset might be foobared
792 _verify_liveness_disable, // no reliable liveness data anymore
793 _verify_regions_disable // no reliable region data here
794 );
795 }
796
797 void ShenandoahVerifier::verify_after_fullgc() {
798 verify_at_safepoint(
799 "After Full GC",
800 _verify_forwarded_none, // all objects are non-forwarded
801 _verify_marked_complete, // all objects are marked in complete bitmap
802 _verify_matrix_conservative, // matrix is conservatively consistent
803 _verify_cset_none, // no cset references
804 _verify_liveness_disable, // no reliable liveness data anymore
805 _verify_regions_notrash_nocset // no trash, no cset
806 );
807 }
808
809 void ShenandoahVerifier::verify_after_degenerated() {
810 verify_at_safepoint(
811 "After Degenerated GC",
812 _verify_forwarded_none, // all objects are non-forwarded
813 _verify_marked_complete, // all objects are marked in complete bitmap
814 _verify_matrix_conservative, // matrix is conservatively consistent
815 _verify_cset_none, // no cset references
816 _verify_liveness_disable, // no reliable liveness data anymore
817 _verify_regions_notrash_nocset // no trash, no cset
818 );
819 }