1 /*
2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 // Deliberately avoids precompiled headers, to break name clashes and accidental dependencies
25
26 #include "gc/shenandoah/brooksPointer.hpp"
27 #include "gc/shenandoah/shenandoahAsserts.hpp"
28 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
29 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
33 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
34 #include "gc/shenandoah/shenandoahVerifier.hpp"
35 #include "memory/allocation.hpp"
36
37 class ShenandoahVerifyOopClosure : public ExtendedOopClosure {
38 private:
39 const char* _phase;
40 ShenandoahVerifier::VerifyOptions _options;
41 ShenandoahVerifierStack* _stack;
42 ShenandoahHeap* _heap;
43 MarkBitMap* _map;
44 ShenandoahLivenessData* _ld;
45 void* _interior_loc;
46 oop _loc;
47
48 public:
49 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
50 const char* phase, ShenandoahVerifier::VerifyOptions options) :
51 _stack(stack), _heap(ShenandoahHeap::heap()), _map(map), _ld(ld), _loc(NULL), _interior_loc(NULL),
52 _phase(phase), _options(options) {};
53
54 private:
55 void verify(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
56 if (!test) {
57 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
58 }
59 }
60
61 template <class T>
62 void do_oop_work(T* p) {
63 T o = RawAccess<>::oop_load(p);
64 if (!CompressedOops::is_null(o)) {
65 oop obj = CompressedOops::decode_not_null(o);
66
67 // Single threaded verification can use faster non-atomic stack and bitmap
68 // methods.
69 //
70 // For performance reasons, only fully verify non-marked field values.
71 // We are here when the host object for *p is already marked. If field value
72 // is marked already, then we have to verify the matrix connection between
73 // host object and field value, regardless.
74
75 HeapWord* addr = (HeapWord*) obj;
76 if (_map->parMark(addr)) {
77 verify_oop_at(p, obj);
78 _stack->push(ShenandoahVerifierTask(obj));
79 } else {
80 verify_matrix(p, obj);
81 }
82 }
83 }
84
85 void verify_oop(oop obj) {
86 // Perform consistency checks with gradually decreasing safety level. This guarantees
87 // that failure report would not try to touch something that was not yet verified to be
88 // safe to process.
89
90 verify(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
91 "oop must be in heap");
92 verify(ShenandoahAsserts::_safe_unknown, obj, check_obj_alignment(obj),
93 "oop must be aligned");
94
95 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
96 Klass* obj_klass = obj->klass_or_null();
97
98 // Verify that obj is not in dead space:
99 {
100 // Do this before touching obj->size()
101 verify(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL,
102 "Object klass pointer should not be NULL");
103 verify(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
104 "Object klass pointer must go to metaspace");
105
106 HeapWord *obj_addr = (HeapWord *) obj;
107 verify(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
108 "Object start should be within the region");
109
110 if (!obj_reg->is_humongous()) {
111 verify(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
112 "Object end should be within the region");
113 } else {
114 size_t humongous_start = obj_reg->region_number();
115 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
116 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
117 verify(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
118 "Humongous object is in continuation that fits it");
119 }
120 }
121
122 // ------------ obj is safe at this point --------------
123
124 verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
125 "Object should be in active region");
126
127 switch (_options._verify_liveness) {
128 case ShenandoahVerifier::_verify_liveness_disable:
129 // skip
130 break;
131 case ShenandoahVerifier::_verify_liveness_complete:
132 Atomic::add(obj->size() + BrooksPointer::word_size(), &_ld[obj_reg->region_number()]);
133 // fallthrough for fast failure for un-live regions:
134 case ShenandoahVerifier::_verify_liveness_conservative:
135 verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
136 "Object must belong to region with live data");
137 break;
138 default:
139 assert(false, "Unhandled liveness verification");
140 }
141 }
142
143 oop fwd = (oop) BrooksPointer::get_raw_unchecked(obj);
144
145 ShenandoahHeapRegion* fwd_reg = NULL;
146
147 if (!oopDesc::unsafe_equals(obj, fwd)) {
148 verify(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
149 "Forwardee must be in heap");
150 verify(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
151 "Forwardee is set");
152 verify(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd),
153 "Forwardee must be aligned");
154
155 // Do this before touching fwd->size()
156 Klass* fwd_klass = fwd->klass_or_null();
157 verify(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL,
158 "Forwardee klass pointer should not be NULL");
159 verify(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
160 "Forwardee klass pointer must go to metaspace");
161 verify(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
162 "Forwardee klass pointer must go to metaspace");
163
164 fwd_reg = _heap->heap_region_containing(fwd);
165
166 // Verify that forwardee is not in the dead space:
167 verify(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
168 "Should have no humongous forwardees");
169
170 HeapWord *fwd_addr = (HeapWord *) fwd;
171 verify(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
172 "Forwardee start should be within the region");
173 verify(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
174 "Forwardee end should be within the region");
175
176 oop fwd2 = (oop) BrooksPointer::get_raw_unchecked(fwd);
177 verify(ShenandoahAsserts::_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2),
178 "Double forwarding");
179 } else {
180 fwd_reg = obj_reg;
181 }
182
183 // ------------ obj and fwd are safe at this point --------------
184
185 switch (_options._verify_marked) {
186 case ShenandoahVerifier::_verify_marked_disable:
187 // skip
188 break;
189 case ShenandoahVerifier::_verify_marked_next:
190 verify(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_next(obj),
191 "Must be marked in next bitmap");
192 break;
193 case ShenandoahVerifier::_verify_marked_complete:
194 verify(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_complete(obj),
195 "Must be marked in complete bitmap");
196 break;
197 default:
198 assert(false, "Unhandled mark verification");
199 }
200
201 switch (_options._verify_forwarded) {
202 case ShenandoahVerifier::_verify_forwarded_disable:
203 // skip
204 break;
205 case ShenandoahVerifier::_verify_forwarded_none: {
206 verify(ShenandoahAsserts::_safe_all, obj, oopDesc::unsafe_equals(obj, fwd),
207 "Should not be forwarded");
208 break;
209 }
210 case ShenandoahVerifier::_verify_forwarded_allow: {
211 if (!oopDesc::unsafe_equals(obj, fwd)) {
212 verify(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg,
213 "Forwardee should be in another region");
214 }
215 break;
216 }
217 default:
218 assert(false, "Unhandled forwarding verification");
219 }
220
221 switch (_options._verify_cset) {
222 case ShenandoahVerifier::_verify_cset_disable:
223 // skip
224 break;
225 case ShenandoahVerifier::_verify_cset_none:
226 verify(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj),
227 "Should not have references to collection set");
228 break;
229 case ShenandoahVerifier::_verify_cset_forwarded:
230 if (_heap->in_collection_set(obj)) {
231 verify(ShenandoahAsserts::_safe_all, obj, !oopDesc::unsafe_equals(obj, fwd),
232 "Object in collection set, should have forwardee");
233 }
234 break;
235 default:
236 assert(false, "Unhandled cset verification");
237 }
238
239 verify_matrix(_interior_loc, obj);
240 }
241
242 void verify_matrix(void* interior, oop obj) {
243 if (!UseShenandoahMatrix || !_heap->is_in(interior)) return;
244 switch (_options._verify_matrix) {
245 case ShenandoahVerifier::_verify_matrix_conservative: {
246 size_t from_idx = _heap->heap_region_index_containing(interior);
247 size_t to_idx = _heap->heap_region_index_containing(obj);
248 _interior_loc = interior;
249 verify(ShenandoahAsserts::_safe_all, obj, _heap->connection_matrix()->is_connected(from_idx, to_idx),
250 "Must be connected");
251 _interior_loc = NULL;
252 break;
253 }
254 case ShenandoahVerifier::_verify_matrix_disable:
255 break;
256 default:
257 assert(false, "Unhandled matrix verification");
258 }
259 }
260
261 public:
262
263 /**
264 * Verify object with known interior reference.
265 * @param p interior reference where the object is referenced from; can be off-heap
266 * @param obj verified object
267 */
268 template <class T>
269 void verify_oop_at(T* p, oop obj) {
270 _interior_loc = p;
271 verify_oop(obj);
272 _interior_loc = NULL;
273 }
274
275 /**
276 * Verify object without known interior reference.
277 * Useful when picking up the object at known offset in heap,
278 * but without knowing what objects reference it.
279 * @param obj verified object
280 */
281 void verify_oop_standalone(oop obj) {
282 _interior_loc = NULL;
283 verify_oop(obj);
284 _interior_loc = NULL;
285 }
286
287 /**
288 * Verify oop fields from this object.
289 * @param obj host object for verified fields
290 */
291 void verify_oops_from(oop obj) {
292 _loc = obj;
293 obj->oop_iterate(this);
294 _loc = NULL;
295 }
296
297 void do_oop(oop* p) { do_oop_work(p); }
298 void do_oop(narrowOop* p) { do_oop_work(p); }
299 };
300
301 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
302 private:
303 size_t _used, _committed, _garbage;
304 public:
305 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
306
307 bool heap_region_do(ShenandoahHeapRegion* r) {
308 _used += r->used();
309 _garbage += r->garbage();
310 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
311 return false;
312 }
313
314 size_t used() { return _used; }
315 size_t committed() { return _committed; }
316 size_t garbage() { return _garbage; }
317 };
318
319 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure {
320 private:
321 ShenandoahHeap* _heap;
322 const char* _phase;
323 ShenandoahVerifier::VerifyRegions _regions;
324 public:
325 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) :
326 _heap(ShenandoahHeap::heap()), _regions(regions), _phase(phase) {};
327
328 void print_failure(ShenandoahHeapRegion* r, const char* label) {
329 ResourceMark rm;
330
331 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label);
332
333 stringStream ss;
334 r->print_on(&ss);
335 msg.append("%s", ss.as_string());
336
337 report_vm_error(__FILE__, __LINE__, msg.buffer());
338 }
339
340 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) {
341 if (!test) {
342 print_failure(r, msg);
343 }
344 }
345
346 bool heap_region_do(ShenandoahHeapRegion* r) {
347 switch (_regions) {
348 case ShenandoahVerifier::_verify_regions_disable:
349 break;
350 case ShenandoahVerifier::_verify_regions_notrash:
351 verify(r, !r->is_trash(),
352 "Should not have trash regions");
353 break;
354 case ShenandoahVerifier::_verify_regions_nocset:
355 verify(r, !r->is_cset(),
356 "Should not have cset regions");
357 break;
358 case ShenandoahVerifier::_verify_regions_notrash_nocset:
359 verify(r, !r->is_trash(),
360 "Should not have trash regions");
361 verify(r, !r->is_cset(),
362 "Should not have cset regions");
363 break;
364 default:
365 ShouldNotReachHere();
366 }
367
368 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
369 "Capacity should match region size");
370
371 verify(r, r->bottom() <= _heap->complete_top_at_mark_start(r->bottom()),
372 "Region top should not be less than bottom");
373
374 verify(r, _heap->complete_top_at_mark_start(r->bottom()) <= r->top(),
375 "Complete TAMS should not be larger than top");
376
377 verify(r, r->get_live_data_bytes() <= r->capacity(),
378 "Live data cannot be larger than capacity");
379
380 verify(r, r->garbage() <= r->capacity(),
381 "Garbage cannot be larger than capacity");
382
383 verify(r, r->used() <= r->capacity(),
384 "Used cannot be larger than capacity");
385
386 verify(r, r->get_shared_allocs() <= r->capacity(),
387 "Shared alloc count should not be larger than capacity");
388
389 verify(r, r->get_tlab_allocs() <= r->capacity(),
390 "TLAB alloc count should not be larger than capacity");
391
392 verify(r, r->get_gclab_allocs() <= r->capacity(),
393 "GCLAB alloc count should not be larger than capacity");
394
395 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
396 "Accurate accounting: shared + TLAB + GCLAB = used");
397
398 verify(r, !r->is_empty() || !r->has_live(),
399 "Empty regions should not have live data");
400
401 verify(r, r->is_cset() == r->in_collection_set(),
402 "Transitional: region flags and collection set agree");
403
404 verify(r, r->is_empty() || r->seqnum_first_alloc() != 0,
405 "Non-empty regions should have first seqnum set");
406
407 verify(r, r->is_empty() || (r->seqnum_first_alloc_mutator() != 0 || r->seqnum_first_alloc_gc() != 0),
408 "Non-empty regions should have first seqnum set to either GC or mutator");
409
410 verify(r, r->is_empty() || r->seqnum_last_alloc() != 0,
411 "Non-empty regions should have last seqnum set");
412
413 verify(r, r->is_empty() || (r->seqnum_last_alloc_mutator() != 0 || r->seqnum_last_alloc_gc() != 0),
414 "Non-empty regions should have last seqnum set to either GC or mutator");
415
416 verify(r, r->seqnum_first_alloc() <= r->seqnum_last_alloc(),
417 "First seqnum should not be greater than last timestamp");
418
419 verify(r, r->seqnum_first_alloc_mutator() <= r->seqnum_last_alloc_mutator(),
420 "First mutator seqnum should not be greater than last seqnum");
421
422 verify(r, r->seqnum_first_alloc_gc() <= r->seqnum_last_alloc_gc(),
423 "First GC seqnum should not be greater than last seqnum");
424
425 return false;
426 }
427 };
428
429 class ShenandoahVerifierReachableTask : public AbstractGangTask {
430 private:
431 const char* _label;
432 ShenandoahRootProcessor* _rp;
433 ShenandoahVerifier::VerifyOptions _options;
434 ShenandoahHeap* _heap;
435 ShenandoahLivenessData* _ld;
436 MarkBitMap* _bitmap;
437 volatile size_t _processed;
438
439 public:
440 ShenandoahVerifierReachableTask(MarkBitMap* bitmap,
441 ShenandoahLivenessData* ld,
442 ShenandoahRootProcessor* rp,
443 const char* label,
444 ShenandoahVerifier::VerifyOptions options) :
445 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"),
446 _heap(ShenandoahHeap::heap()), _rp(rp), _ld(ld), _bitmap(bitmap), _processed(0),
447 _label(label), _options(options) {};
448
449 size_t processed() {
450 return _processed;
451 }
452
453 virtual void work(uint worker_id) {
454 ResourceMark rm;
455 ShenandoahVerifierStack stack;
456
457 // On level 2, we need to only check the roots once.
458 // On level 3, we want to check the roots, and seed the local stack.
459 // It is a lesser evil to accept multiple root scans at level 3, because
460 // extended parallelism would buy us out.
461 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0))
462 || (ShenandoahVerifyLevel >= 3)) {
463 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
464 ShenandoahMessageBuffer("%s, Roots", _label),
465 _options);
466 _rp->process_all_roots_slow(&cl);
467 }
468
469 size_t processed = 0;
470
471 if (ShenandoahVerifyLevel >= 3) {
472 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
473 ShenandoahMessageBuffer("%s, Reachable", _label),
474 _options);
475 while (!stack.is_empty()) {
476 processed++;
477 ShenandoahVerifierTask task = stack.pop();
478 cl.verify_oops_from(task.obj());
479 }
480 }
481
482 Atomic::add(processed, &_processed);
483 }
484 };
485
486 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask {
487 private:
488 const char* _label;
489 ShenandoahVerifier::VerifyOptions _options;
490 ShenandoahHeap *_heap;
491 MarkBitMap* _bitmap;
492 ShenandoahLivenessData* _ld;
493 volatile size_t _claimed;
494 volatile size_t _processed;
495
496 public:
497 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
498 ShenandoahLivenessData* ld,
499 const char* label,
500 ShenandoahVerifier::VerifyOptions options) :
501 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"),
502 _heap(ShenandoahHeap::heap()), _ld(ld), _bitmap(bitmap), _claimed(0), _processed(0),
503 _label(label), _options(options) {};
504
505 size_t processed() {
506 return _processed;
507 }
508
509 virtual void work(uint worker_id) {
510 ShenandoahVerifierStack stack;
511 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
512 ShenandoahMessageBuffer("%s, Marked", _label),
513 _options);
514
515 while (true) {
516 size_t v = Atomic::add(1u, &_claimed) - 1;
517 if (v < _heap->num_regions()) {
518 ShenandoahHeapRegion* r = _heap->get_region(v);
519 if (!r->is_humongous() && !r->is_trash()) {
520 work_regular(r, stack, cl);
521 } else if (r->is_humongous_start()) {
522 work_humongous(r, stack, cl);
523 }
524 } else {
525 break;
526 }
527 }
528 }
529
530 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
531 size_t processed = 0;
532 HeapWord* obj = r->bottom() + BrooksPointer::word_size();
533 if (_heap->is_marked_complete((oop)obj)) {
534 verify_and_follow(obj, stack, cl, &processed);
535 }
536 Atomic::add(processed, &_processed);
537 }
538
539 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
540 size_t processed = 0;
541 MarkBitMap* mark_bit_map = _heap->complete_mark_bit_map();
542 HeapWord* tams = _heap->complete_top_at_mark_start(r->bottom());
543
544 // Bitmaps, before TAMS
545 if (tams > r->bottom()) {
546 HeapWord* start = r->bottom() + BrooksPointer::word_size();
547 HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams);
548
549 while (addr < tams) {
550 verify_and_follow(addr, stack, cl, &processed);
551 addr += BrooksPointer::word_size();
552 if (addr < tams) {
553 addr = mark_bit_map->getNextMarkedWordAddress(addr, tams);
554 }
555 }
556 }
557
558 // Size-based, after TAMS
559 {
560 HeapWord* limit = r->top();
561 HeapWord* addr = tams + BrooksPointer::word_size();
562
563 while (addr < limit) {
564 verify_and_follow(addr, stack, cl, &processed);
565 addr += oop(addr)->size() + BrooksPointer::word_size();
566 }
567 }
568
569 Atomic::add(processed, &_processed);
570 }
571
572 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
573 if (!_bitmap->parMark(addr)) return;
574
575 // Verify the object itself:
576 oop obj = oop(addr);
577 cl.verify_oop_standalone(obj);
578
579 // Verify everything reachable from that object too, hopefully realizing
580 // everything was already marked, and never touching further:
581 cl.verify_oops_from(obj);
582 (*processed)++;
583
584 while (!stack.is_empty()) {
585 ShenandoahVerifierTask task = stack.pop();
586 cl.verify_oops_from(task.obj());
587 (*processed)++;
588 }
589 }
590 };
591
592 void ShenandoahVerifier::verify_at_safepoint(const char *label,
593 VerifyForwarded forwarded, VerifyMarked marked,
594 VerifyMatrix matrix, VerifyCollectionSet cset,
595 VerifyLiveness liveness, VerifyRegions regions) {
596 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
597 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
598
599 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
600 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
601
602 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel);
603
604 // Heap size checks
605 {
606 ShenandoahHeapLocker lock(_heap->lock());
607
608 ShenandoahCalculateRegionStatsClosure cl;
609 _heap->heap_region_iterate(&cl);
610 size_t heap_used = _heap->used();
611 guarantee(cl.used() == heap_used,
612 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "K, regions-used = " SIZE_FORMAT "K",
613 label, heap_used/K, cl.used()/K);
614
615 size_t heap_committed = _heap->committed();
616 guarantee(cl.committed() == heap_committed,
617 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "K, regions-committed = " SIZE_FORMAT "K",
618 label, heap_committed/K, cl.committed()/K);
619 }
620
621 // Internal heap region checks
622 if (ShenandoahVerifyLevel >= 1) {
623 ShenandoahVerifyHeapRegionClosure cl(label, regions);
624 _heap->heap_region_iterate(&cl,
625 /* skip_cset = */ false,
626 /* skip_humongous_cont = */ false);
627 }
628
629 OrderAccess::fence();
630 _heap->make_tlabs_parsable(false);
631
632 // Allocate temporary bitmap for storing marking wavefront:
633 MemRegion mr = MemRegion(_verification_bit_map->startWord(), _verification_bit_map->endWord());
634 _verification_bit_map->clear_range_large(mr);
635
636 // Allocate temporary array for storing liveness data
637 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC);
638 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0);
639
640 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, matrix, cset, liveness, regions);
641
642 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
643 // This verifies what application can see, since it only cares about reachable objects.
644 size_t count_reachable = 0;
645 if (ShenandoahVerifyLevel >= 2) {
646 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(),
647 ShenandoahPhaseTimings::_num_phases); // no need for stats
648
649 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options);
650 _heap->workers()->run_task(&task);
651 count_reachable = task.processed();
652 }
653
654 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
655 // not the application, can see during the region scans. There is no reason to process the objects
656 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
657 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
658 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
659 // version
660
661 size_t count_marked = 0;
662 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
663 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options);
664 _heap->workers()->run_task(&task);
665 count_marked = task.processed();
666 } else {
667 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_next || marked == _verify_marked_disable, "Should be");
668 }
669
670 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
671 // marked objects.
672
673 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
674 for (size_t i = 0; i < _heap->num_regions(); i++) {
675 ShenandoahHeapRegion* r = _heap->get_region(i);
676
677 juint verf_live = 0;
678 if (r->is_humongous()) {
679 // For humongous objects, test if start region is marked live, and if so,
680 // all humongous regions in that chain have live data equal to their "used".
681 juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]);
682 if (start_live > 0) {
683 verf_live = (juint)(r->used() / HeapWordSize);
684 }
685 } else {
686 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]);
687 }
688
689 size_t reg_live = r->get_live_data_words();
690 if (reg_live != verf_live) {
691 ResourceMark rm;
692 stringStream ss;
693 r->print_on(&ss);
694 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s",
695 label, reg_live, verf_live, ss.as_string());
696 }
697 }
698 }
699
700 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)",
701 label, ShenandoahVerifyLevel, count_reachable, count_marked);
702
703 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld);
704 }
705
706 void ShenandoahVerifier::verify_generic(VerifyOption vo) {
707 verify_at_safepoint(
708 "Generic Verification",
709 _verify_forwarded_allow, // conservatively allow forwarded
710 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
711 _verify_matrix_disable, // matrix can be inconsistent here
712 _verify_cset_disable, // cset may be inconsistent
713 _verify_liveness_disable, // no reliable liveness data
714 _verify_regions_disable // no reliable region data
715 );
716 }
717
718 void ShenandoahVerifier::verify_before_concmark() {
719 if (_heap->has_forwarded_objects()) {
720 verify_at_safepoint(
721 "Before Mark",
722 _verify_forwarded_allow, // may have forwarded references
723 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
724 _verify_matrix_disable, // matrix is foobared
725 _verify_cset_forwarded, // allow forwarded references to cset
726 _verify_liveness_disable, // no reliable liveness data
727 _verify_regions_notrash // no trash regions
728 );
729 } else {
730 verify_at_safepoint(
731 "Before Mark",
732 _verify_forwarded_none, // UR should have fixed up
733 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
734 _verify_matrix_conservative, // UR should have fixed matrix
735 _verify_cset_none, // UR should have fixed this
736 _verify_liveness_disable, // no reliable liveness data
737 _verify_regions_notrash // no trash regions
738 );
739 }
740 }
741
742 void ShenandoahVerifier::verify_after_concmark() {
743 verify_at_safepoint(
744 "After Mark",
745 _verify_forwarded_none, // no forwarded references
746 _verify_marked_complete, // bitmaps as precise as we can get
747 _verify_matrix_disable, // matrix might be foobared
748 _verify_cset_none, // no references to cset anymore
749 _verify_liveness_complete, // liveness data must be complete here
750 _verify_regions_disable // trash regions not yet recycled
751 );
752 }
753
754 void ShenandoahVerifier::verify_before_evacuation() {
755 // Evacuation is always preceded by mark, but we want to have a sanity check after
756 // selecting the collection set, and (immediate) regions recycling
757 verify_at_safepoint(
758 "Before Evacuation",
759 _verify_forwarded_none, // no forwarded references
760 _verify_marked_complete, // walk over marked objects too
761 _verify_matrix_disable, // skip, verified after mark
762 _verify_cset_disable, // skip, verified after mark
763 _verify_liveness_disable, // skip, verified after mark
764 _verify_regions_disable // trash regions not yet recycled
765 );
766 }
767
768 void ShenandoahVerifier::verify_after_evacuation() {
769 verify_at_safepoint(
770 "After Evacuation",
771 _verify_forwarded_allow, // objects are still forwarded
772 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
773 _verify_matrix_disable, // matrix is inconsistent here
774 _verify_cset_forwarded, // all cset refs are fully forwarded
775 _verify_liveness_disable, // no reliable liveness data anymore
776 _verify_regions_notrash // trash regions have been recycled already
777 );
778 }
779
780 void ShenandoahVerifier::verify_before_updaterefs() {
781 verify_at_safepoint(
782 "Before Updating References",
783 _verify_forwarded_allow, // forwarded references allowed
784 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
785 _verify_matrix_disable, // matrix is inconsistent here
786 _verify_cset_forwarded, // all cset refs are fully forwarded
787 _verify_liveness_disable, // no reliable liveness data anymore
788 _verify_regions_notrash // trash regions have been recycled already
789 );
790 }
791
792 void ShenandoahVerifier::verify_after_updaterefs() {
793 verify_at_safepoint(
794 "After Updating References",
795 _verify_forwarded_none, // no forwarded references
796 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
797 _verify_matrix_conservative, // matrix is conservatively consistent
798 _verify_cset_none, // no cset references, all updated
799 _verify_liveness_disable, // no reliable liveness data anymore
800 _verify_regions_nocset // no cset regions, trash regions have appeared
801 );
802 }
803
804 void ShenandoahVerifier::verify_after_degenerated() {
805 verify_at_safepoint(
806 "After Degenerated GC",
807 _verify_forwarded_none, // all objects are non-forwarded
808 _verify_marked_complete, // all objects are marked in complete bitmap
809 _verify_matrix_conservative, // matrix is conservatively consistent
810 _verify_cset_none, // no cset references
811 _verify_liveness_disable, // no reliable liveness data anymore
812 _verify_regions_notrash_nocset // no trash, no cset
813 );
814 }
815
816 void ShenandoahVerifier::verify_before_traversal() {
817 verify_at_safepoint(
818 "Before Traversal",
819 _verify_forwarded_none, // cannot have forwarded objects
820 _verify_marked_disable, // bitmaps are not relevant before traversal
821 _verify_matrix_disable, // matrix is not used in traversal
822 _verify_cset_none, // no cset references before traversal
823 _verify_liveness_disable, // no reliable liveness data anymore
824 _verify_regions_notrash_nocset // no trash and no cset regions
825 );
826 }
827
828 void ShenandoahVerifier::verify_after_traversal() {
829 verify_at_safepoint(
830 "After Traversal",
831 _verify_forwarded_none, // cannot have forwarded objects
832 _verify_marked_disable, // We only have partial marking info after traversal
833 _verify_matrix_disable, // matrix is conservatively consistent
834 _verify_cset_none, // no cset references left after traversal
835 _verify_liveness_disable, // liveness data is not collected for new allocations
836 _verify_regions_nocset // no cset regions, trash regions allowed
837 );
838 }
839
840 void ShenandoahVerifier::verify_before_fullgc() {
841 verify_at_safepoint(
842 "Before Full GC",
843 _verify_forwarded_allow, // can have forwarded objects
844 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
845 _verify_matrix_disable, // matrix might be foobared
846 _verify_cset_disable, // cset might be foobared
847 _verify_liveness_disable, // no reliable liveness data anymore
848 _verify_regions_disable // no reliable region data here
849 );
850 }
851
852 void ShenandoahVerifier::verify_after_fullgc() {
853 verify_at_safepoint(
854 "After Full GC",
855 _verify_forwarded_none, // all objects are non-forwarded
856 _verify_marked_complete, // all objects are marked in complete bitmap
857 _verify_matrix_conservative, // matrix is conservatively consistent
858 _verify_cset_none, // no cset references
859 _verify_liveness_disable, // no reliable liveness data anymore
860 _verify_regions_notrash_nocset // no trash, no cset
861 );
862 }
863