1 /*
2 * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shenandoah/shenandoahAsserts.hpp"
28 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
29 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
30 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
33 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
34 #include "gc/shenandoah/shenandoahUtils.hpp"
35 #include "gc/shenandoah/shenandoahVerifier.hpp"
36 #include "memory/allocation.hpp"
37 #include "memory/iterator.inline.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "oops/compressedOops.inline.hpp"
40 #include "runtime/atomic.hpp"
41 #include "runtime/orderAccess.hpp"
42 #include "utilities/align.hpp"
43
44 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
45 #ifdef verify_oop
46 #undef verify_oop
47 #endif
48
49 class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
50 private:
51 const char* _phase;
52 ShenandoahVerifier::VerifyOptions _options;
53 ShenandoahVerifierStack* _stack;
54 ShenandoahHeap* _heap;
55 MarkBitMap* _map;
56 ShenandoahLivenessData* _ld;
57 void* _interior_loc;
58 oop _loc;
59
60 public:
61 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
62 const char* phase, ShenandoahVerifier::VerifyOptions options) :
63 _phase(phase),
64 _options(options),
65 _stack(stack),
66 _heap(ShenandoahHeap::heap()),
67 _map(map),
68 _ld(ld),
69 _interior_loc(NULL),
70 _loc(NULL) { }
71
72 private:
73 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
74 if (!test) {
75 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
76 }
77 }
78
79 template <class T>
80 void do_oop_work(T* p) {
81 T o = RawAccess<>::oop_load(p);
82 if (!CompressedOops::is_null(o)) {
83 oop obj = CompressedOops::decode_not_null(o);
84 assert_object_is_in_heap(p, obj);
85
86 // Single threaded verification can use faster non-atomic stack and bitmap
87 // methods.
88 //
89 // For performance reasons, only fully verify non-marked field values.
90 // We are here when the host object for *p is already marked.
91
92 HeapWord* addr = (HeapWord*) obj;
93 if (_map->par_mark(addr)) {
94 verify_oop_at(p, obj);
95 _stack->push(ShenandoahVerifierTask(obj));
96 }
97 }
98 }
99
100 void verify_oop(oop obj) {
101 // Perform consistency checks with gradually decreasing safety level. This guarantees
102 // that failure report would not try to touch something that was not yet verified to be
103 // safe to process.
104
105 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
106 "oop must be in heap");
107 check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
108 "oop must be aligned");
109
110 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
111 Klass* obj_klass = obj->klass_or_null();
112
113 // Verify that obj is not in dead space:
114 {
115 // Do this before touching obj->size()
116 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL,
117 "Object klass pointer should not be NULL");
118 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
119 "Object klass pointer must go to metaspace");
120
121 HeapWord *obj_addr = (HeapWord *) obj;
122 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
123 "Object start should be within the region");
124
125 if (!obj_reg->is_humongous()) {
126 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
127 "Object end should be within the region");
128 } else {
129 size_t humongous_start = obj_reg->region_number();
130 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
131 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
132 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
133 "Humongous object is in continuation that fits it");
134 }
135 }
136
137 // ------------ obj is safe at this point --------------
138
139 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
140 "Object should be in active region");
141
142 switch (_options._verify_liveness) {
143 case ShenandoahVerifier::_verify_liveness_disable:
144 // skip
145 break;
146 case ShenandoahVerifier::_verify_liveness_complete:
147 Atomic::add(&_ld[obj_reg->region_number()], (uint) obj->size());
148 // fallthrough for fast failure for un-live regions:
149 case ShenandoahVerifier::_verify_liveness_conservative:
150 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
151 "Object must belong to region with live data");
152 break;
153 default:
154 assert(false, "Unhandled liveness verification");
155 }
156 }
157
158 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
159
160 ShenandoahHeapRegion* fwd_reg = NULL;
161
162 if (obj != fwd) {
163 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
164 "Forwardee must be in heap");
165 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
166 "Forwardee is set");
167 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
168 "Forwardee must be aligned");
169
170 // Do this before touching fwd->size()
171 Klass* fwd_klass = fwd->klass_or_null();
172 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL,
173 "Forwardee klass pointer should not be NULL");
174 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
175 "Forwardee klass pointer must go to metaspace");
176 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
177 "Forwardee klass pointer must go to metaspace");
178
179 fwd_reg = _heap->heap_region_containing(fwd);
180
181 // Verify that forwardee is not in the dead space:
182 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
183 "Should have no humongous forwardees");
184
185 HeapWord *fwd_addr = (HeapWord *) fwd;
186 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
187 "Forwardee start should be within the region");
188 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
189 "Forwardee end should be within the region");
190
191 oop fwd2 = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
192 check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
193 "Double forwarding");
194 } else {
195 fwd_reg = obj_reg;
196 }
197
198 // ------------ obj and fwd are safe at this point --------------
199
200 switch (_options._verify_marked) {
201 case ShenandoahVerifier::_verify_marked_disable:
202 // skip
203 break;
204 case ShenandoahVerifier::_verify_marked_incomplete:
205 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
206 "Must be marked in incomplete bitmap");
207 break;
208 case ShenandoahVerifier::_verify_marked_complete:
209 check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj),
210 "Must be marked in complete bitmap");
211 break;
212 default:
213 assert(false, "Unhandled mark verification");
214 }
215
216 switch (_options._verify_forwarded) {
217 case ShenandoahVerifier::_verify_forwarded_disable:
218 // skip
219 break;
220 case ShenandoahVerifier::_verify_forwarded_none: {
221 check(ShenandoahAsserts::_safe_all, obj, (obj == fwd),
222 "Should not be forwarded");
223 break;
224 }
225 case ShenandoahVerifier::_verify_forwarded_allow: {
226 if (obj != fwd) {
227 check(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg,
228 "Forwardee should be in another region");
229 }
230 break;
231 }
232 default:
233 assert(false, "Unhandled forwarding verification");
234 }
235
236 switch (_options._verify_cset) {
237 case ShenandoahVerifier::_verify_cset_disable:
238 // skip
239 break;
240 case ShenandoahVerifier::_verify_cset_none:
241 check(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj),
242 "Should not have references to collection set");
243 break;
244 case ShenandoahVerifier::_verify_cset_forwarded:
245 if (_heap->in_collection_set(obj)) {
246 check(ShenandoahAsserts::_safe_all, obj, (obj != fwd),
247 "Object in collection set, should have forwardee");
248 }
249 break;
250 default:
251 assert(false, "Unhandled cset verification");
252 }
253
254 }
255
256 public:
257 /**
258 * Verify object with known interior reference.
259 * @param p interior reference where the object is referenced from; can be off-heap
260 * @param obj verified object
261 */
262 template <class T>
263 void verify_oop_at(T* p, oop obj) {
264 _interior_loc = p;
265 verify_oop(obj);
266 _interior_loc = NULL;
267 }
268
269 /**
270 * Verify object without known interior reference.
271 * Useful when picking up the object at known offset in heap,
272 * but without knowing what objects reference it.
273 * @param obj verified object
274 */
275 void verify_oop_standalone(oop obj) {
276 _interior_loc = NULL;
277 verify_oop(obj);
278 _interior_loc = NULL;
279 }
280
281 /**
282 * Verify oop fields from this object.
283 * @param obj host object for verified fields
284 */
285 void verify_oops_from(oop obj) {
286 _loc = obj;
287 obj->oop_iterate(this);
288 _loc = NULL;
289 }
290
291 virtual void do_oop(oop* p) { do_oop_work(p); }
292 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
293 };
294
295 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
296 private:
297 size_t _used, _committed, _garbage;
298 public:
299 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
300
301 void heap_region_do(ShenandoahHeapRegion* r) {
302 _used += r->used();
303 _garbage += r->garbage();
304 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
305 }
306
307 size_t used() { return _used; }
308 size_t committed() { return _committed; }
309 size_t garbage() { return _garbage; }
310 };
311
312 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure {
313 private:
314 ShenandoahHeap* _heap;
315 const char* _phase;
316 ShenandoahVerifier::VerifyRegions _regions;
317 public:
318 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) :
319 _heap(ShenandoahHeap::heap()),
320 _phase(phase),
321 _regions(regions) {};
322
323 void print_failure(ShenandoahHeapRegion* r, const char* label) {
324 ResourceMark rm;
325
326 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label);
327
328 stringStream ss;
329 r->print_on(&ss);
330 msg.append("%s", ss.as_string());
331
332 report_vm_error(__FILE__, __LINE__, msg.buffer());
333 }
334
335 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) {
336 if (!test) {
337 print_failure(r, msg);
338 }
339 }
340
341 void heap_region_do(ShenandoahHeapRegion* r) {
342 switch (_regions) {
343 case ShenandoahVerifier::_verify_regions_disable:
344 break;
345 case ShenandoahVerifier::_verify_regions_notrash:
346 verify(r, !r->is_trash(),
347 "Should not have trash regions");
348 break;
349 case ShenandoahVerifier::_verify_regions_nocset:
350 verify(r, !r->is_cset(),
351 "Should not have cset regions");
352 break;
353 case ShenandoahVerifier::_verify_regions_notrash_nocset:
354 verify(r, !r->is_trash(),
355 "Should not have trash regions");
356 verify(r, !r->is_cset(),
357 "Should not have cset regions");
358 break;
359 default:
360 ShouldNotReachHere();
361 }
362
363 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
364 "Capacity should match region size");
365
366 verify(r, r->bottom() <= r->top(),
367 "Region top should not be less than bottom");
368
369 verify(r, r->bottom() <= _heap->marking_context()->top_at_mark_start(r),
370 "Region TAMS should not be less than bottom");
371
372 verify(r, _heap->marking_context()->top_at_mark_start(r) <= r->top(),
373 "Complete TAMS should not be larger than top");
374
375 verify(r, r->get_live_data_bytes() <= r->capacity(),
376 "Live data cannot be larger than capacity");
377
378 verify(r, r->garbage() <= r->capacity(),
379 "Garbage cannot be larger than capacity");
380
381 verify(r, r->used() <= r->capacity(),
382 "Used cannot be larger than capacity");
383
384 verify(r, r->get_shared_allocs() <= r->capacity(),
385 "Shared alloc count should not be larger than capacity");
386
387 verify(r, r->get_tlab_allocs() <= r->capacity(),
388 "TLAB alloc count should not be larger than capacity");
389
390 verify(r, r->get_gclab_allocs() <= r->capacity(),
391 "GCLAB alloc count should not be larger than capacity");
392
393 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
394 "Accurate accounting: shared + TLAB + GCLAB = used");
395
396 verify(r, !r->is_empty() || !r->has_live(),
397 "Empty regions should not have live data");
398
399 verify(r, r->is_cset() == _heap->collection_set()->is_in(r),
400 "Transitional: region flags and collection set agree");
401
402 verify(r, r->is_empty() || r->seqnum_first_alloc() != 0,
403 "Non-empty regions should have first seqnum set");
404
405 verify(r, r->is_empty() || (r->seqnum_first_alloc_mutator() != 0 || r->seqnum_first_alloc_gc() != 0),
406 "Non-empty regions should have first seqnum set to either GC or mutator");
407
408 verify(r, r->is_empty() || r->seqnum_last_alloc() != 0,
409 "Non-empty regions should have last seqnum set");
410
411 verify(r, r->is_empty() || (r->seqnum_last_alloc_mutator() != 0 || r->seqnum_last_alloc_gc() != 0),
412 "Non-empty regions should have last seqnum set to either GC or mutator");
413
414 verify(r, r->seqnum_first_alloc() <= r->seqnum_last_alloc(),
415 "First seqnum should not be greater than last timestamp");
416
417 verify(r, r->seqnum_first_alloc_mutator() <= r->seqnum_last_alloc_mutator(),
418 "First mutator seqnum should not be greater than last seqnum");
419
420 verify(r, r->seqnum_first_alloc_gc() <= r->seqnum_last_alloc_gc(),
421 "First GC seqnum should not be greater than last seqnum");
422 }
423 };
424
425 class ShenandoahVerifierReachableTask : public AbstractGangTask {
426 private:
427 const char* _label;
428 ShenandoahRootVerifier* _verifier;
429 ShenandoahVerifier::VerifyOptions _options;
430 ShenandoahHeap* _heap;
431 ShenandoahLivenessData* _ld;
432 MarkBitMap* _bitmap;
433 volatile size_t _processed;
434
435 public:
436 ShenandoahVerifierReachableTask(MarkBitMap* bitmap,
437 ShenandoahLivenessData* ld,
438 ShenandoahRootVerifier* verifier,
439 const char* label,
440 ShenandoahVerifier::VerifyOptions options) :
441 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"),
442 _label(label),
443 _verifier(verifier),
444 _options(options),
445 _heap(ShenandoahHeap::heap()),
446 _ld(ld),
447 _bitmap(bitmap),
448 _processed(0) {};
449
450 size_t processed() {
451 return _processed;
452 }
453
454 virtual void work(uint worker_id) {
455 ResourceMark rm;
456 ShenandoahVerifierStack stack;
457
458 // On level 2, we need to only check the roots once.
459 // On level 3, we want to check the roots, and seed the local stack.
460 // It is a lesser evil to accept multiple root scans at level 3, because
461 // extended parallelism would buy us out.
462 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0))
463 || (ShenandoahVerifyLevel >= 3)) {
464 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
465 ShenandoahMessageBuffer("%s, Roots", _label),
466 _options);
467 if (_heap->unload_classes()) {
468 _verifier->strong_roots_do(&cl);
469 } else {
470 _verifier->roots_do(&cl);
471 }
472 }
473
474 size_t processed = 0;
475
476 if (ShenandoahVerifyLevel >= 3) {
477 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
478 ShenandoahMessageBuffer("%s, Reachable", _label),
479 _options);
480 while (!stack.is_empty()) {
481 processed++;
482 ShenandoahVerifierTask task = stack.pop();
483 cl.verify_oops_from(task.obj());
484 }
485 }
486
487 Atomic::add(&_processed, processed);
488 }
489 };
490
491 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask {
492 private:
493 const char* _label;
494 ShenandoahVerifier::VerifyOptions _options;
495 ShenandoahHeap *_heap;
496 MarkBitMap* _bitmap;
497 ShenandoahLivenessData* _ld;
498 volatile size_t _claimed;
499 volatile size_t _processed;
500
501 public:
502 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
503 ShenandoahLivenessData* ld,
504 const char* label,
505 ShenandoahVerifier::VerifyOptions options) :
506 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"),
507 _label(label),
508 _options(options),
509 _heap(ShenandoahHeap::heap()),
510 _bitmap(bitmap),
511 _ld(ld),
512 _claimed(0),
513 _processed(0) {};
514
515 size_t processed() {
516 return _processed;
517 }
518
519 virtual void work(uint worker_id) {
520 ShenandoahVerifierStack stack;
521 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
522 ShenandoahMessageBuffer("%s, Marked", _label),
523 _options);
524
525 while (true) {
526 size_t v = Atomic::add(&_claimed, 1u) - 1;
527 if (v < _heap->num_regions()) {
528 ShenandoahHeapRegion* r = _heap->get_region(v);
529 if (!r->is_humongous() && !r->is_trash()) {
530 work_regular(r, stack, cl);
531 } else if (r->is_humongous_start()) {
532 work_humongous(r, stack, cl);
533 }
534 } else {
535 break;
536 }
537 }
538 }
539
540 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
541 size_t processed = 0;
542 HeapWord* obj = r->bottom();
543 if (_heap->complete_marking_context()->is_marked((oop)obj)) {
544 verify_and_follow(obj, stack, cl, &processed);
545 }
546 Atomic::add(&_processed, processed);
547 }
548
549 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
550 size_t processed = 0;
551 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
552 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
553
554 // Bitmaps, before TAMS
555 if (tams > r->bottom()) {
556 HeapWord* start = r->bottom();
557 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
558
559 while (addr < tams) {
560 verify_and_follow(addr, stack, cl, &processed);
561 addr += 1;
562 if (addr < tams) {
563 addr = mark_bit_map->get_next_marked_addr(addr, tams);
564 }
565 }
566 }
567
568 // Size-based, after TAMS
569 {
570 HeapWord* limit = r->top();
571 HeapWord* addr = tams;
572
573 while (addr < limit) {
574 verify_and_follow(addr, stack, cl, &processed);
575 addr += oop(addr)->size();
576 }
577 }
578
579 Atomic::add(&_processed, processed);
580 }
581
582 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
583 if (!_bitmap->par_mark(addr)) return;
584
585 // Verify the object itself:
586 oop obj = oop(addr);
587 cl.verify_oop_standalone(obj);
588
589 // Verify everything reachable from that object too, hopefully realizing
590 // everything was already marked, and never touching further:
591 cl.verify_oops_from(obj);
592 (*processed)++;
593
594 while (!stack.is_empty()) {
595 ShenandoahVerifierTask task = stack.pop();
596 cl.verify_oops_from(task.obj());
597 (*processed)++;
598 }
599 }
600 };
601
602 class VerifyThreadGCState : public ThreadClosure {
603 private:
604 const char* _label;
605 char _expected;
606
607 public:
608 VerifyThreadGCState(const char* label, char expected) : _expected(expected) {}
609 void do_thread(Thread* t) {
610 char actual = ShenandoahThreadLocalData::gc_state(t);
611 if (actual != _expected) {
612 fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual);
613 }
614 }
615 };
616
617 class ShenandoahGCStateResetter : public StackObj {
618 private:
619 ShenandoahHeap* const _heap;
620 char _gc_state;
621
622 public:
623 ShenandoahGCStateResetter() : _heap(ShenandoahHeap::heap()) {
624 _gc_state = _heap->gc_state();
625 _heap->_gc_state.clear();
626 }
627
628 ~ShenandoahGCStateResetter() {
629 _heap->_gc_state.set(_gc_state);
630 assert(_heap->gc_state() == _gc_state, "Should be restored");
631 }
632 };
633
634 void ShenandoahVerifier::verify_at_safepoint(const char *label,
635 VerifyForwarded forwarded, VerifyMarked marked,
636 VerifyCollectionSet cset,
637 VerifyLiveness liveness, VerifyRegions regions,
638 VerifyGCState gcstate,
639 VerifyWeakRoots weak_roots) {
640 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
641 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
642
643 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
644 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
645
646 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel);
647
648 // GC state checks
649 {
650 char expected = -1;
651 bool enabled;
652 switch (gcstate) {
653 case _verify_gcstate_disable:
654 enabled = false;
655 break;
656 case _verify_gcstate_forwarded:
657 enabled = true;
658 expected = ShenandoahHeap::HAS_FORWARDED;
659 break;
660 case _verify_gcstate_evacuation:
661 enabled = true;
662 expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION;
663 break;
664 case _verify_gcstate_stable:
665 enabled = true;
666 expected = ShenandoahHeap::STABLE;
667 break;
668 default:
669 enabled = false;
670 assert(false, "Unhandled gc-state verification");
671 }
672
673 if (enabled) {
674 char actual = _heap->gc_state();
675 if (actual != expected) {
676 fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual);
677 }
678
679 VerifyThreadGCState vtgcs(label, expected);
680 Threads::java_threads_do(&vtgcs);
681 }
682 }
683
684 // Deactivate barriers temporarily: Verifier wants plain heap accesses
685 ShenandoahGCStateResetter resetter;
686
687 // Heap size checks
688 {
689 ShenandoahHeapLocker lock(_heap->lock());
690
691 ShenandoahCalculateRegionStatsClosure cl;
692 _heap->heap_region_iterate(&cl);
693 size_t heap_used = _heap->used();
694 guarantee(cl.used() == heap_used,
695 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s",
696 label,
697 byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used),
698 byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used()));
699
700 size_t heap_committed = _heap->committed();
701 guarantee(cl.committed() == heap_committed,
702 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "%s, regions-committed = " SIZE_FORMAT "%s",
703 label,
704 byte_size_in_proper_unit(heap_committed), proper_unit_for_byte_size(heap_committed),
705 byte_size_in_proper_unit(cl.committed()), proper_unit_for_byte_size(cl.committed()));
706 }
707
708 // Internal heap region checks
709 if (ShenandoahVerifyLevel >= 1) {
710 ShenandoahVerifyHeapRegionClosure cl(label, regions);
711 _heap->heap_region_iterate(&cl);
712 }
713
714 OrderAccess::fence();
715 _heap->make_parsable(false);
716
717 // Allocate temporary bitmap for storing marking wavefront:
718 _verification_bit_map->clear();
719
720 // Allocate temporary array for storing liveness data
721 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC);
722 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0);
723
724 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, cset, liveness, regions, gcstate);
725
726 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
727 // This verifies what application can see, since it only cares about reachable objects.
728 size_t count_reachable = 0;
729 if (ShenandoahVerifyLevel >= 2) {
730 ShenandoahRootVerifier verifier;
731 switch (weak_roots) {
732 case _verify_serial_weak_roots:
733 verifier.excludes(ShenandoahRootVerifier::ConcurrentWeakRoots);
734 break;
735 case _verify_concurrent_weak_roots:
736 verifier.excludes(ShenandoahRootVerifier::SerialWeakRoots);
737 break;
738 case _verify_all_weak_roots:
739 break;
740 default:
741 ShouldNotReachHere();
742 }
743
744 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &verifier, label, options);
745 _heap->workers()->run_task(&task);
746 count_reachable = task.processed();
747 }
748
749 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
750 // not the application, can see during the region scans. There is no reason to process the objects
751 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
752 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
753 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
754 // version
755
756 size_t count_marked = 0;
757 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
758 guarantee(_heap->marking_context()->is_complete(), "Marking context should be complete");
759 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options);
760 _heap->workers()->run_task(&task);
761 count_marked = task.processed();
762 } else {
763 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_incomplete || marked == _verify_marked_disable, "Should be");
764 }
765
766 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
767 // marked objects.
768
769 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
770 for (size_t i = 0; i < _heap->num_regions(); i++) {
771 ShenandoahHeapRegion* r = _heap->get_region(i);
772
773 juint verf_live = 0;
774 if (r->is_humongous()) {
775 // For humongous objects, test if start region is marked live, and if so,
776 // all humongous regions in that chain have live data equal to their "used".
777 juint start_live = Atomic::load_acquire(&ld[r->humongous_start_region()->region_number()]);
778 if (start_live > 0) {
779 verf_live = (juint)(r->used() / HeapWordSize);
780 }
781 } else {
782 verf_live = Atomic::load_acquire(&ld[r->region_number()]);
783 }
784
785 size_t reg_live = r->get_live_data_words();
786 if (reg_live != verf_live) {
787 ResourceMark rm;
788 stringStream ss;
789 r->print_on(&ss);
790 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s",
791 label, reg_live, verf_live, ss.as_string());
792 }
793 }
794 }
795
796 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)",
797 label, ShenandoahVerifyLevel, count_reachable, count_marked);
798
799 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld);
800 }
801
802 void ShenandoahVerifier::verify_generic(VerifyOption vo) {
803 verify_at_safepoint(
804 "Generic Verification",
805 _verify_forwarded_allow, // conservatively allow forwarded
806 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
807 _verify_cset_disable, // cset may be inconsistent
808 _verify_liveness_disable, // no reliable liveness data
809 _verify_regions_disable, // no reliable region data
810 _verify_gcstate_disable, // no data about gcstate
811 _verify_all_weak_roots
812 );
813 }
814
815 void ShenandoahVerifier::verify_before_concmark() {
816 if (_heap->has_forwarded_objects()) {
817 verify_at_safepoint(
818 "Before Mark",
819 _verify_forwarded_allow, // may have forwarded references
820 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
821 _verify_cset_forwarded, // allow forwarded references to cset
822 _verify_liveness_disable, // no reliable liveness data
823 _verify_regions_notrash, // no trash regions
824 _verify_gcstate_forwarded, // there are forwarded objects
825 _verify_all_weak_roots
826 );
827 } else {
828 verify_at_safepoint(
829 "Before Mark",
830 _verify_forwarded_none, // UR should have fixed up
831 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
832 _verify_cset_none, // UR should have fixed this
833 _verify_liveness_disable, // no reliable liveness data
834 _verify_regions_notrash, // no trash regions
835 _verify_gcstate_stable, // there are no forwarded objects
836 _verify_all_weak_roots
837 );
838 }
839 }
840
841 void ShenandoahVerifier::verify_after_concmark() {
842 verify_at_safepoint(
843 "After Mark",
844 _verify_forwarded_none, // no forwarded references
845 _verify_marked_complete, // bitmaps as precise as we can get
846 _verify_cset_none, // no references to cset anymore
847 _verify_liveness_complete, // liveness data must be complete here
848 _verify_regions_disable, // trash regions not yet recycled
849 _verify_gcstate_stable, // mark should have stabilized the heap
850 _verify_all_weak_roots
851 );
852 }
853
854 void ShenandoahVerifier::verify_before_evacuation() {
855 // Concurrent weak roots are evacuated during concurrent phase
856 VerifyWeakRoots verify_weak_roots = ShenandoahConcurrentRoots::should_do_concurrent_class_unloading() ?
857 _verify_serial_weak_roots :
858 _verify_all_weak_roots;
859
860 verify_at_safepoint(
861 "Before Evacuation",
862 _verify_forwarded_none, // no forwarded references
863 _verify_marked_complete, // walk over marked objects too
864 _verify_cset_disable, // non-forwarded references to cset expected
865 _verify_liveness_complete, // liveness data must be complete here
866 _verify_regions_disable, // trash regions not yet recycled
867 _verify_gcstate_stable, // mark should have stabilized the heap
868 verify_weak_roots
869 );
870 }
871
872 void ShenandoahVerifier::verify_during_evacuation() {
873 // Concurrent weak roots are evacuated during concurrent phase
874 VerifyWeakRoots verify_weak_roots = ShenandoahConcurrentRoots::should_do_concurrent_class_unloading() ?
875 _verify_serial_weak_roots :
876 _verify_all_weak_roots;
877
878 verify_at_safepoint(
879 "During Evacuation",
880 _verify_forwarded_allow, // some forwarded references are allowed
881 _verify_marked_disable, // walk only roots
882 _verify_cset_disable, // some cset references are not forwarded yet
883 _verify_liveness_disable, // liveness data might be already stale after pre-evacs
884 _verify_regions_disable, // trash regions not yet recycled
885 _verify_gcstate_evacuation, // evacuation is in progress
886 verify_weak_roots
887 );
888 }
889
890 void ShenandoahVerifier::verify_after_evacuation() {
891 verify_at_safepoint(
892 "After Evacuation",
893 _verify_forwarded_allow, // objects are still forwarded
894 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
895 _verify_cset_forwarded, // all cset refs are fully forwarded
896 _verify_liveness_disable, // no reliable liveness data anymore
897 _verify_regions_notrash, // trash regions have been recycled already
898 _verify_gcstate_forwarded, // evacuation produced some forwarded objects
899 _verify_all_weak_roots
900 );
901 }
902
903 void ShenandoahVerifier::verify_before_updaterefs() {
904 verify_at_safepoint(
905 "Before Updating References",
906 _verify_forwarded_allow, // forwarded references allowed
907 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
908 _verify_cset_forwarded, // all cset refs are fully forwarded
909 _verify_liveness_disable, // no reliable liveness data anymore
910 _verify_regions_notrash, // trash regions have been recycled already
911 _verify_gcstate_forwarded, // evacuation should have produced some forwarded objects
912 _verify_all_weak_roots
913 );
914 }
915
916 void ShenandoahVerifier::verify_after_updaterefs() {
917 verify_at_safepoint(
918 "After Updating References",
919 _verify_forwarded_none, // no forwarded references
920 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
921 _verify_cset_none, // no cset references, all updated
922 _verify_liveness_disable, // no reliable liveness data anymore
923 _verify_regions_nocset, // no cset regions, trash regions have appeared
924 _verify_gcstate_stable, // update refs had cleaned up forwarded objects
925 _verify_all_weak_roots
926 );
927 }
928
929 void ShenandoahVerifier::verify_after_degenerated() {
930 verify_at_safepoint(
931 "After Degenerated GC",
932 _verify_forwarded_none, // all objects are non-forwarded
933 _verify_marked_complete, // all objects are marked in complete bitmap
934 _verify_cset_none, // no cset references
935 _verify_liveness_disable, // no reliable liveness data anymore
936 _verify_regions_notrash_nocset, // no trash, no cset
937 _verify_gcstate_stable, // degenerated refs had cleaned up forwarded objects
938 _verify_all_weak_roots
939 );
940 }
941
942 void ShenandoahVerifier::verify_before_traversal() {
943 verify_at_safepoint(
944 "Before Traversal",
945 _verify_forwarded_none, // cannot have forwarded objects
946 _verify_marked_disable, // bitmaps are not relevant before traversal
947 _verify_cset_none, // no cset references before traversal
948 _verify_liveness_disable, // no reliable liveness data anymore
949 _verify_regions_notrash_nocset, // no trash and no cset regions
950 _verify_gcstate_stable, // nothing forwarded before traversal
951 _verify_all_weak_roots
952 );
953 }
954
955 void ShenandoahVerifier::verify_after_traversal() {
956 verify_at_safepoint(
957 "After Traversal",
958 _verify_forwarded_none, // cannot have forwarded objects
959 _verify_marked_complete, // should have complete marking after traversal
960 _verify_cset_none, // no cset references left after traversal
961 _verify_liveness_disable, // liveness data is not collected for new allocations
962 _verify_regions_nocset, // no cset regions, trash regions allowed
963 _verify_gcstate_stable, // nothing forwarded after traversal
964 _verify_all_weak_roots
965 );
966 }
967
968 void ShenandoahVerifier::verify_before_fullgc() {
969 verify_at_safepoint(
970 "Before Full GC",
971 _verify_forwarded_allow, // can have forwarded objects
972 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
973 _verify_cset_disable, // cset might be foobared
974 _verify_liveness_disable, // no reliable liveness data anymore
975 _verify_regions_disable, // no reliable region data here
976 _verify_gcstate_disable, // no reliable gcstate data
977 _verify_all_weak_roots
978 );
979 }
980
981 void ShenandoahVerifier::verify_after_fullgc() {
982 verify_at_safepoint(
983 "After Full GC",
984 _verify_forwarded_none, // all objects are non-forwarded
985 _verify_marked_complete, // all objects are marked in complete bitmap
986 _verify_cset_none, // no cset references
987 _verify_liveness_disable, // no reliable liveness data anymore
988 _verify_regions_notrash_nocset, // no trash, no cset
989 _verify_gcstate_stable, // full gc cleaned up everything
990 _verify_all_weak_roots
991 );
992 }
993
994 class ShenandoahVerifyNoForwared : public OopClosure {
995 private:
996 template <class T>
997 void do_oop_work(T* p) {
998 T o = RawAccess<>::oop_load(p);
999 if (!CompressedOops::is_null(o)) {
1000 oop obj = CompressedOops::decode_not_null(o);
1001 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
1002 if (obj != fwd) {
1003 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
1004 "Verify Roots", "Should not be forwarded", __FILE__, __LINE__);
1005 }
1006 }
1007 }
1008
1009 public:
1010 void do_oop(narrowOop* p) { do_oop_work(p); }
1011 void do_oop(oop* p) { do_oop_work(p); }
1012 };
1013
1014 class ShenandoahVerifyInToSpaceClosure : public OopClosure {
1015 private:
1016 template <class T>
1017 void do_oop_work(T* p) {
1018 T o = RawAccess<>::oop_load(p);
1019 if (!CompressedOops::is_null(o)) {
1020 oop obj = CompressedOops::decode_not_null(o);
1021 ShenandoahHeap* heap = ShenandoahHeap::heap_no_check();
1022
1023 if (!heap->marking_context()->is_marked(obj)) {
1024 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
1025 "Verify Roots In To-Space", "Should be marked", __FILE__, __LINE__);
1026 }
1027
1028 if (heap->in_collection_set(obj)) {
1029 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
1030 "Verify Roots In To-Space", "Should not be in collection set", __FILE__, __LINE__);
1031 }
1032
1033 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
1034 if (obj != fwd) {
1035 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
1036 "Verify Roots In To-Space", "Should not be forwarded", __FILE__, __LINE__);
1037 }
1038 }
1039 }
1040
1041 public:
1042 void do_oop(narrowOop* p) { do_oop_work(p); }
1043 void do_oop(oop* p) { do_oop_work(p); }
1044 };
1045
1046 void ShenandoahVerifier::verify_roots_in_to_space() {
1047 ShenandoahRootVerifier verifier;
1048 ShenandoahVerifyInToSpaceClosure cl;
1049 verifier.oops_do(&cl);
1050 }
1051
1052 void ShenandoahVerifier::verify_roots_in_to_space_except(ShenandoahRootVerifier::RootTypes types) {
1053 ShenandoahRootVerifier verifier;
1054 verifier.excludes(types);
1055 ShenandoahVerifyInToSpaceClosure cl;
1056 verifier.oops_do(&cl);
1057 }
1058
1059 void ShenandoahVerifier::verify_roots_no_forwarded() {
1060 ShenandoahRootVerifier verifier;
1061 ShenandoahVerifyNoForwared cl;
1062 verifier.oops_do(&cl);
1063 }
1064
1065 void ShenandoahVerifier::verify_roots_no_forwarded_except(ShenandoahRootVerifier::RootTypes types) {
1066 ShenandoahRootVerifier verifier;
1067 verifier.excludes(types);
1068 ShenandoahVerifyNoForwared cl;
1069 verifier.oops_do(&cl);
1070 }