1 /*
2 * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shenandoah/shenandoahAsserts.hpp"
27 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
28 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
31 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
32 #include "gc/shenandoah/shenandoahUtils.hpp"
33 #include "gc/shenandoah/shenandoahVerifier.hpp"
34 #include "memory/allocation.hpp"
35 #include "memory/iterator.inline.hpp"
36 #include "memory/resourceArea.hpp"
37 #include "oops/compressedOops.inline.hpp"
38 #include "runtime/atomic.hpp"
39 #include "utilities/align.hpp"
40
41 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
42 #ifdef verify_oop
43 #undef verify_oop
44 #endif
45
46 class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
47 private:
48 const char* _phase;
49 ShenandoahVerifier::VerifyOptions _options;
50 ShenandoahVerifierStack* _stack;
51 ShenandoahHeap* _heap;
52 MarkBitMap* _map;
53 ShenandoahLivenessData* _ld;
54 void* _interior_loc;
55 oop _loc;
56
57 public:
58 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
59 const char* phase, ShenandoahVerifier::VerifyOptions options) :
60 _phase(phase),
61 _options(options),
62 _stack(stack),
63 _heap(ShenandoahHeap::heap()),
64 _map(map),
65 _ld(ld),
66 _interior_loc(NULL),
67 _loc(NULL) { }
68
69 private:
70 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
71 if (!test) {
72 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
73 }
74 }
75
76 template <class T>
77 void do_oop_work(T* p) {
78 T o = RawAccess<>::oop_load(p);
79 if (!CompressedOops::is_null(o)) {
80 oop obj = CompressedOops::decode_not_null(o);
81
82 // Single threaded verification can use faster non-atomic stack and bitmap
83 // methods.
84 //
85 // For performance reasons, only fully verify non-marked field values.
86 // We are here when the host object for *p is already marked.
87
88 HeapWord* addr = (HeapWord*) obj;
89 if (_map->par_mark(addr)) {
90 verify_oop_at(p, obj);
91 _stack->push(ShenandoahVerifierTask(obj));
92 }
93 }
94 }
95
96 void verify_oop(oop obj) {
97 // Perform consistency checks with gradually decreasing safety level. This guarantees
98 // that failure report would not try to touch something that was not yet verified to be
99 // safe to process.
100
101 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
102 "oop must be in heap");
103 check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
104 "oop must be aligned");
105
106 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
107 Klass* obj_klass = obj->klass_or_null();
108
109 // Verify that obj is not in dead space:
110 {
111 // Do this before touching obj->size()
112 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL,
113 "Object klass pointer should not be NULL");
114 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
115 "Object klass pointer must go to metaspace");
116
117 HeapWord *obj_addr = (HeapWord *) obj;
118 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
119 "Object start should be within the region");
120
121 if (!obj_reg->is_humongous()) {
122 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
123 "Object end should be within the region");
124 } else {
125 size_t humongous_start = obj_reg->region_number();
126 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
127 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
128 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
129 "Humongous object is in continuation that fits it");
130 }
131 }
132
133 // ------------ obj is safe at this point --------------
134
135 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
136 "Object should be in active region");
137
138 switch (_options._verify_liveness) {
139 case ShenandoahVerifier::_verify_liveness_disable:
140 // skip
141 break;
142 case ShenandoahVerifier::_verify_liveness_complete:
143 Atomic::add(&_ld[obj_reg->region_number()], (uint) obj->size());
144 // fallthrough for fast failure for un-live regions:
145 case ShenandoahVerifier::_verify_liveness_conservative:
146 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
147 "Object must belong to region with live data");
148 break;
149 default:
150 assert(false, "Unhandled liveness verification");
151 }
152 }
153
154 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
155
156 ShenandoahHeapRegion* fwd_reg = NULL;
157
158 if (obj != fwd) {
159 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
160 "Forwardee must be in heap");
161 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
162 "Forwardee is set");
163 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
164 "Forwardee must be aligned");
165
166 // Do this before touching fwd->size()
167 Klass* fwd_klass = fwd->klass_or_null();
168 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL,
169 "Forwardee klass pointer should not be NULL");
170 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
171 "Forwardee klass pointer must go to metaspace");
172 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
173 "Forwardee klass pointer must go to metaspace");
174
175 fwd_reg = _heap->heap_region_containing(fwd);
176
177 // Verify that forwardee is not in the dead space:
178 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
179 "Should have no humongous forwardees");
180
181 HeapWord *fwd_addr = (HeapWord *) fwd;
182 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
183 "Forwardee start should be within the region");
184 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
185 "Forwardee end should be within the region");
186
187 oop fwd2 = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
188 check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
189 "Double forwarding");
190 } else {
191 fwd_reg = obj_reg;
192 }
193
194 // ------------ obj and fwd are safe at this point --------------
195
196 switch (_options._verify_marked) {
197 case ShenandoahVerifier::_verify_marked_disable:
198 // skip
199 break;
200 case ShenandoahVerifier::_verify_marked_incomplete:
201 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
202 "Must be marked in incomplete bitmap");
203 break;
204 case ShenandoahVerifier::_verify_marked_complete:
205 check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj),
206 "Must be marked in complete bitmap");
207 break;
208 default:
209 assert(false, "Unhandled mark verification");
210 }
211
212 switch (_options._verify_forwarded) {
213 case ShenandoahVerifier::_verify_forwarded_disable:
214 // skip
215 break;
216 case ShenandoahVerifier::_verify_forwarded_none: {
217 check(ShenandoahAsserts::_safe_all, obj, (obj == fwd),
218 "Should not be forwarded");
219 break;
220 }
221 case ShenandoahVerifier::_verify_forwarded_allow: {
222 if (obj != fwd) {
223 check(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg,
224 "Forwardee should be in another region");
225 }
226 break;
227 }
228 default:
229 assert(false, "Unhandled forwarding verification");
230 }
231
232 switch (_options._verify_cset) {
233 case ShenandoahVerifier::_verify_cset_disable:
234 // skip
235 break;
236 case ShenandoahVerifier::_verify_cset_none:
237 check(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj),
238 "Should not have references to collection set");
239 break;
240 case ShenandoahVerifier::_verify_cset_forwarded:
241 if (_heap->in_collection_set(obj)) {
242 check(ShenandoahAsserts::_safe_all, obj, (obj != fwd),
243 "Object in collection set, should have forwardee");
244 }
245 break;
246 default:
247 assert(false, "Unhandled cset verification");
248 }
249
250 }
251
252 public:
253 /**
254 * Verify object with known interior reference.
255 * @param p interior reference where the object is referenced from; can be off-heap
256 * @param obj verified object
257 */
258 template <class T>
259 void verify_oop_at(T* p, oop obj) {
260 _interior_loc = p;
261 verify_oop(obj);
262 _interior_loc = NULL;
263 }
264
265 /**
266 * Verify object without known interior reference.
267 * Useful when picking up the object at known offset in heap,
268 * but without knowing what objects reference it.
269 * @param obj verified object
270 */
271 void verify_oop_standalone(oop obj) {
272 _interior_loc = NULL;
273 verify_oop(obj);
274 _interior_loc = NULL;
275 }
276
277 /**
278 * Verify oop fields from this object.
279 * @param obj host object for verified fields
280 */
281 void verify_oops_from(oop obj) {
282 _loc = obj;
283 obj->oop_iterate(this);
284 _loc = NULL;
285 }
286
287 virtual void do_oop(oop* p) { do_oop_work(p); }
288 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
289 };
290
291 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
292 private:
293 size_t _used, _committed, _garbage;
294 public:
295 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
296
297 void heap_region_do(ShenandoahHeapRegion* r) {
298 _used += r->used();
299 _garbage += r->garbage();
300 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
301 }
302
303 size_t used() { return _used; }
304 size_t committed() { return _committed; }
305 size_t garbage() { return _garbage; }
306 };
307
308 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure {
309 private:
310 ShenandoahHeap* _heap;
311 const char* _phase;
312 ShenandoahVerifier::VerifyRegions _regions;
313 public:
314 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) :
315 _heap(ShenandoahHeap::heap()),
316 _phase(phase),
317 _regions(regions) {};
318
319 void print_failure(ShenandoahHeapRegion* r, const char* label) {
320 ResourceMark rm;
321
322 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label);
323
324 stringStream ss;
325 r->print_on(&ss);
326 msg.append("%s", ss.as_string());
327
328 report_vm_error(__FILE__, __LINE__, msg.buffer());
329 }
330
331 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) {
332 if (!test) {
333 print_failure(r, msg);
334 }
335 }
336
337 void heap_region_do(ShenandoahHeapRegion* r) {
338 switch (_regions) {
339 case ShenandoahVerifier::_verify_regions_disable:
340 break;
341 case ShenandoahVerifier::_verify_regions_notrash:
342 verify(r, !r->is_trash(),
343 "Should not have trash regions");
344 break;
345 case ShenandoahVerifier::_verify_regions_nocset:
346 verify(r, !r->is_cset(),
347 "Should not have cset regions");
348 break;
349 case ShenandoahVerifier::_verify_regions_notrash_nocset:
350 verify(r, !r->is_trash(),
351 "Should not have trash regions");
352 verify(r, !r->is_cset(),
353 "Should not have cset regions");
354 break;
355 default:
356 ShouldNotReachHere();
357 }
358
359 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
360 "Capacity should match region size");
361
362 verify(r, r->bottom() <= r->top(),
363 "Region top should not be less than bottom");
364
365 verify(r, r->bottom() <= _heap->marking_context()->top_at_mark_start(r),
366 "Region TAMS should not be less than bottom");
367
368 verify(r, _heap->marking_context()->top_at_mark_start(r) <= r->top(),
369 "Complete TAMS should not be larger than top");
370
371 verify(r, r->get_live_data_bytes() <= r->capacity(),
372 "Live data cannot be larger than capacity");
373
374 verify(r, r->garbage() <= r->capacity(),
375 "Garbage cannot be larger than capacity");
376
377 verify(r, r->used() <= r->capacity(),
378 "Used cannot be larger than capacity");
379
380 verify(r, r->get_shared_allocs() <= r->capacity(),
381 "Shared alloc count should not be larger than capacity");
382
383 verify(r, r->get_tlab_allocs() <= r->capacity(),
384 "TLAB alloc count should not be larger than capacity");
385
386 verify(r, r->get_gclab_allocs() <= r->capacity(),
387 "GCLAB alloc count should not be larger than capacity");
388
389 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
390 "Accurate accounting: shared + TLAB + GCLAB = used");
391
392 verify(r, !r->is_empty() || !r->has_live(),
393 "Empty regions should not have live data");
394
395 verify(r, r->is_cset() == _heap->collection_set()->is_in(r),
396 "Transitional: region flags and collection set agree");
397
398 verify(r, r->is_empty() || r->seqnum_first_alloc() != 0,
399 "Non-empty regions should have first seqnum set");
400
401 verify(r, r->is_empty() || (r->seqnum_first_alloc_mutator() != 0 || r->seqnum_first_alloc_gc() != 0),
402 "Non-empty regions should have first seqnum set to either GC or mutator");
403
404 verify(r, r->is_empty() || r->seqnum_last_alloc() != 0,
405 "Non-empty regions should have last seqnum set");
406
407 verify(r, r->is_empty() || (r->seqnum_last_alloc_mutator() != 0 || r->seqnum_last_alloc_gc() != 0),
408 "Non-empty regions should have last seqnum set to either GC or mutator");
409
410 verify(r, r->seqnum_first_alloc() <= r->seqnum_last_alloc(),
411 "First seqnum should not be greater than last timestamp");
412
413 verify(r, r->seqnum_first_alloc_mutator() <= r->seqnum_last_alloc_mutator(),
414 "First mutator seqnum should not be greater than last seqnum");
415
416 verify(r, r->seqnum_first_alloc_gc() <= r->seqnum_last_alloc_gc(),
417 "First GC seqnum should not be greater than last seqnum");
418 }
419 };
420
421 class ShenandoahVerifierReachableTask : public AbstractGangTask {
422 private:
423 const char* _label;
424 ShenandoahRootVerifier* _verifier;
425 ShenandoahVerifier::VerifyOptions _options;
426 ShenandoahHeap* _heap;
427 ShenandoahLivenessData* _ld;
428 MarkBitMap* _bitmap;
429 volatile size_t _processed;
430
431 public:
432 ShenandoahVerifierReachableTask(MarkBitMap* bitmap,
433 ShenandoahLivenessData* ld,
434 ShenandoahRootVerifier* verifier,
435 const char* label,
436 ShenandoahVerifier::VerifyOptions options) :
437 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"),
438 _label(label),
439 _verifier(verifier),
440 _options(options),
441 _heap(ShenandoahHeap::heap()),
442 _ld(ld),
443 _bitmap(bitmap),
444 _processed(0) {};
445
446 size_t processed() {
447 return _processed;
448 }
449
450 virtual void work(uint worker_id) {
451 ResourceMark rm;
452 ShenandoahVerifierStack stack;
453
454 // On level 2, we need to only check the roots once.
455 // On level 3, we want to check the roots, and seed the local stack.
456 // It is a lesser evil to accept multiple root scans at level 3, because
457 // extended parallelism would buy us out.
458 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0))
459 || (ShenandoahVerifyLevel >= 3)) {
460 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
461 ShenandoahMessageBuffer("%s, Roots", _label),
462 _options);
463 if (_heap->unload_classes()) {
464 _verifier->strong_roots_do(&cl);
465 } else {
466 _verifier->roots_do(&cl);
467 }
468 }
469
470 size_t processed = 0;
471
472 if (ShenandoahVerifyLevel >= 3) {
473 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
474 ShenandoahMessageBuffer("%s, Reachable", _label),
475 _options);
476 while (!stack.is_empty()) {
477 processed++;
478 ShenandoahVerifierTask task = stack.pop();
479 cl.verify_oops_from(task.obj());
480 }
481 }
482
483 Atomic::add(&_processed, processed);
484 }
485 };
486
487 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask {
488 private:
489 const char* _label;
490 ShenandoahVerifier::VerifyOptions _options;
491 ShenandoahHeap *_heap;
492 MarkBitMap* _bitmap;
493 ShenandoahLivenessData* _ld;
494 volatile size_t _claimed;
495 volatile size_t _processed;
496
497 public:
498 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
499 ShenandoahLivenessData* ld,
500 const char* label,
501 ShenandoahVerifier::VerifyOptions options) :
502 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"),
503 _label(label),
504 _options(options),
505 _heap(ShenandoahHeap::heap()),
506 _bitmap(bitmap),
507 _ld(ld),
508 _claimed(0),
509 _processed(0) {};
510
511 size_t processed() {
512 return _processed;
513 }
514
515 virtual void work(uint worker_id) {
516 ShenandoahVerifierStack stack;
517 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
518 ShenandoahMessageBuffer("%s, Marked", _label),
519 _options);
520
521 while (true) {
522 size_t v = Atomic::add(&_claimed, 1u) - 1;
523 if (v < _heap->num_regions()) {
524 ShenandoahHeapRegion* r = _heap->get_region(v);
525 if (!r->is_humongous() && !r->is_trash()) {
526 work_regular(r, stack, cl);
527 } else if (r->is_humongous_start()) {
528 work_humongous(r, stack, cl);
529 }
530 } else {
531 break;
532 }
533 }
534 }
535
536 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
537 size_t processed = 0;
538 HeapWord* obj = r->bottom();
539 if (_heap->complete_marking_context()->is_marked((oop)obj)) {
540 verify_and_follow(obj, stack, cl, &processed);
541 }
542 Atomic::add(&_processed, processed);
543 }
544
545 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
546 size_t processed = 0;
547 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
548 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
549
550 // Bitmaps, before TAMS
551 if (tams > r->bottom()) {
552 HeapWord* start = r->bottom();
553 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
554
555 while (addr < tams) {
556 verify_and_follow(addr, stack, cl, &processed);
557 addr += 1;
558 if (addr < tams) {
559 addr = mark_bit_map->get_next_marked_addr(addr, tams);
560 }
561 }
562 }
563
564 // Size-based, after TAMS
565 {
566 HeapWord* limit = r->top();
567 HeapWord* addr = tams;
568
569 while (addr < limit) {
570 verify_and_follow(addr, stack, cl, &processed);
571 addr += oop(addr)->size();
572 }
573 }
574
575 Atomic::add(&_processed, processed);
576 }
577
578 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
579 if (!_bitmap->par_mark(addr)) return;
580
581 // Verify the object itself:
582 oop obj = oop(addr);
583 cl.verify_oop_standalone(obj);
584
585 // Verify everything reachable from that object too, hopefully realizing
586 // everything was already marked, and never touching further:
587 cl.verify_oops_from(obj);
588 (*processed)++;
589
590 while (!stack.is_empty()) {
591 ShenandoahVerifierTask task = stack.pop();
592 cl.verify_oops_from(task.obj());
593 (*processed)++;
594 }
595 }
596 };
597
598 class VerifyThreadGCState : public ThreadClosure {
599 private:
600 const char* _label;
601 char _expected;
602
603 public:
604 VerifyThreadGCState(const char* label, char expected) : _expected(expected) {}
605 void do_thread(Thread* t) {
606 char actual = ShenandoahThreadLocalData::gc_state(t);
607 if (actual != _expected) {
608 fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual);
609 }
610 }
611 };
612
613 class ShenandoahGCStateResetter : public StackObj {
614 private:
615 ShenandoahHeap* const _heap;
616 char _gc_state;
617
618 public:
619 ShenandoahGCStateResetter() : _heap(ShenandoahHeap::heap()) {
620 _gc_state = _heap->gc_state();
621 _heap->_gc_state.clear();
622 }
623
624 ~ShenandoahGCStateResetter() {
625 _heap->_gc_state.set(_gc_state);
626 assert(_heap->gc_state() == _gc_state, "Should be restored");
627 }
628 };
629
630 void ShenandoahVerifier::verify_at_safepoint(const char *label,
631 VerifyForwarded forwarded, VerifyMarked marked,
632 VerifyCollectionSet cset,
633 VerifyLiveness liveness, VerifyRegions regions,
634 VerifyGCState gcstate) {
635 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
636 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
637
638 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
639 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
640
641 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel);
642
643 // GC state checks
644 {
645 char expected = -1;
646 bool enabled;
647 switch (gcstate) {
648 case _verify_gcstate_disable:
649 enabled = false;
650 break;
651 case _verify_gcstate_forwarded:
652 enabled = true;
653 expected = ShenandoahHeap::HAS_FORWARDED;
654 break;
655 case _verify_gcstate_evacuation:
656 enabled = true;
657 expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION;
658 break;
659 case _verify_gcstate_stable:
660 enabled = true;
661 expected = ShenandoahHeap::STABLE;
662 break;
663 default:
664 enabled = false;
665 assert(false, "Unhandled gc-state verification");
666 }
667
668 if (enabled) {
669 char actual = _heap->gc_state();
670 if (actual != expected) {
671 fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual);
672 }
673
674 VerifyThreadGCState vtgcs(label, expected);
675 Threads::java_threads_do(&vtgcs);
676 }
677 }
678
679 // Deactivate barriers temporarily: Verifier wants plain heap accesses
680 ShenandoahGCStateResetter resetter;
681
682 // Heap size checks
683 {
684 ShenandoahHeapLocker lock(_heap->lock());
685
686 ShenandoahCalculateRegionStatsClosure cl;
687 _heap->heap_region_iterate(&cl);
688 size_t heap_used = _heap->used();
689 guarantee(cl.used() == heap_used,
690 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s",
691 label,
692 byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used),
693 byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used()));
694
695 size_t heap_committed = _heap->committed();
696 guarantee(cl.committed() == heap_committed,
697 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "%s, regions-committed = " SIZE_FORMAT "%s",
698 label,
699 byte_size_in_proper_unit(heap_committed), proper_unit_for_byte_size(heap_committed),
700 byte_size_in_proper_unit(cl.committed()), proper_unit_for_byte_size(cl.committed()));
701 }
702
703 // Internal heap region checks
704 if (ShenandoahVerifyLevel >= 1) {
705 ShenandoahVerifyHeapRegionClosure cl(label, regions);
706 _heap->heap_region_iterate(&cl);
707 }
708
709 OrderAccess::fence();
710 _heap->make_parsable(false);
711
712 // Allocate temporary bitmap for storing marking wavefront:
713 _verification_bit_map->clear();
714
715 // Allocate temporary array for storing liveness data
716 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC);
717 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0);
718
719 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, cset, liveness, regions, gcstate);
720
721 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
722 // This verifies what application can see, since it only cares about reachable objects.
723 size_t count_reachable = 0;
724 if (ShenandoahVerifyLevel >= 2) {
725 ShenandoahRootVerifier verifier;
726
727 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &verifier, label, options);
728 _heap->workers()->run_task(&task);
729 count_reachable = task.processed();
730 }
731
732 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
733 // not the application, can see during the region scans. There is no reason to process the objects
734 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
735 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
736 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
737 // version
738
739 size_t count_marked = 0;
740 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
741 guarantee(_heap->marking_context()->is_complete(), "Marking context should be complete");
742 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options);
743 _heap->workers()->run_task(&task);
744 count_marked = task.processed();
745 } else {
746 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_incomplete || marked == _verify_marked_disable, "Should be");
747 }
748
749 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
750 // marked objects.
751
752 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
753 for (size_t i = 0; i < _heap->num_regions(); i++) {
754 ShenandoahHeapRegion* r = _heap->get_region(i);
755
756 juint verf_live = 0;
757 if (r->is_humongous()) {
758 // For humongous objects, test if start region is marked live, and if so,
759 // all humongous regions in that chain have live data equal to their "used".
760 juint start_live = Atomic::load_acquire(&ld[r->humongous_start_region()->region_number()]);
761 if (start_live > 0) {
762 verf_live = (juint)(r->used() / HeapWordSize);
763 }
764 } else {
765 verf_live = Atomic::load_acquire(&ld[r->region_number()]);
766 }
767
768 size_t reg_live = r->get_live_data_words();
769 if (reg_live != verf_live) {
770 ResourceMark rm;
771 stringStream ss;
772 r->print_on(&ss);
773 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s",
774 label, reg_live, verf_live, ss.as_string());
775 }
776 }
777 }
778
779 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)",
780 label, ShenandoahVerifyLevel, count_reachable, count_marked);
781
782 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld);
783 }
784
785 void ShenandoahVerifier::verify_generic(VerifyOption vo) {
786 verify_at_safepoint(
787 "Generic Verification",
788 _verify_forwarded_allow, // conservatively allow forwarded
789 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
790 _verify_cset_disable, // cset may be inconsistent
791 _verify_liveness_disable, // no reliable liveness data
792 _verify_regions_disable, // no reliable region data
793 _verify_gcstate_disable // no data about gcstate
794 );
795 }
796
797 void ShenandoahVerifier::verify_before_concmark() {
798 if (_heap->has_forwarded_objects()) {
799 verify_at_safepoint(
800 "Before Mark",
801 _verify_forwarded_allow, // may have forwarded references
802 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
803 _verify_cset_forwarded, // allow forwarded references to cset
804 _verify_liveness_disable, // no reliable liveness data
805 _verify_regions_notrash, // no trash regions
806 _verify_gcstate_forwarded // there are forwarded objects
807 );
808 } else {
809 verify_at_safepoint(
810 "Before Mark",
811 _verify_forwarded_none, // UR should have fixed up
812 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
813 _verify_cset_none, // UR should have fixed this
814 _verify_liveness_disable, // no reliable liveness data
815 _verify_regions_notrash, // no trash regions
816 _verify_gcstate_stable // there are no forwarded objects
817 );
818 }
819 }
820
821 void ShenandoahVerifier::verify_after_concmark() {
822 verify_at_safepoint(
823 "After Mark",
824 _verify_forwarded_none, // no forwarded references
825 _verify_marked_complete, // bitmaps as precise as we can get
826 _verify_cset_none, // no references to cset anymore
827 _verify_liveness_complete, // liveness data must be complete here
828 _verify_regions_disable, // trash regions not yet recycled
829 _verify_gcstate_stable // mark should have stabilized the heap
830 );
831 }
832
833 void ShenandoahVerifier::verify_before_evacuation() {
834 verify_at_safepoint(
835 "Before Evacuation",
836 _verify_forwarded_none, // no forwarded references
837 _verify_marked_complete, // walk over marked objects too
838 _verify_cset_disable, // non-forwarded references to cset expected
839 _verify_liveness_complete, // liveness data must be complete here
840 _verify_regions_disable, // trash regions not yet recycled
841 _verify_gcstate_stable // mark should have stabilized the heap
842 );
843 }
844
845 void ShenandoahVerifier::verify_during_evacuation() {
846 verify_at_safepoint(
847 "During Evacuation",
848 _verify_forwarded_allow, // some forwarded references are allowed
849 _verify_marked_disable, // walk only roots
850 _verify_cset_disable, // some cset references are not forwarded yet
851 _verify_liveness_disable, // liveness data might be already stale after pre-evacs
852 _verify_regions_disable, // trash regions not yet recycled
853 _verify_gcstate_evacuation // evacuation is in progress
854 );
855 }
856
857 void ShenandoahVerifier::verify_after_evacuation() {
858 verify_at_safepoint(
859 "After Evacuation",
860 _verify_forwarded_allow, // objects are still forwarded
861 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
862 _verify_cset_forwarded, // all cset refs are fully forwarded
863 _verify_liveness_disable, // no reliable liveness data anymore
864 _verify_regions_notrash, // trash regions have been recycled already
865 _verify_gcstate_forwarded // evacuation produced some forwarded objects
866 );
867 }
868
869 void ShenandoahVerifier::verify_before_updaterefs() {
870 verify_at_safepoint(
871 "Before Updating References",
872 _verify_forwarded_allow, // forwarded references allowed
873 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
874 _verify_cset_forwarded, // all cset refs are fully forwarded
875 _verify_liveness_disable, // no reliable liveness data anymore
876 _verify_regions_notrash, // trash regions have been recycled already
877 _verify_gcstate_forwarded // evacuation should have produced some forwarded objects
878 );
879 }
880
881 void ShenandoahVerifier::verify_after_updaterefs() {
882 verify_at_safepoint(
883 "After Updating References",
884 _verify_forwarded_none, // no forwarded references
885 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
886 _verify_cset_none, // no cset references, all updated
887 _verify_liveness_disable, // no reliable liveness data anymore
888 _verify_regions_nocset, // no cset regions, trash regions have appeared
889 _verify_gcstate_stable // update refs had cleaned up forwarded objects
890 );
891 }
892
893 void ShenandoahVerifier::verify_after_degenerated() {
894 verify_at_safepoint(
895 "After Degenerated GC",
896 _verify_forwarded_none, // all objects are non-forwarded
897 _verify_marked_complete, // all objects are marked in complete bitmap
898 _verify_cset_none, // no cset references
899 _verify_liveness_disable, // no reliable liveness data anymore
900 _verify_regions_notrash_nocset, // no trash, no cset
901 _verify_gcstate_stable // degenerated refs had cleaned up forwarded objects
902 );
903 }
904
905 void ShenandoahVerifier::verify_before_traversal() {
906 verify_at_safepoint(
907 "Before Traversal",
908 _verify_forwarded_none, // cannot have forwarded objects
909 _verify_marked_disable, // bitmaps are not relevant before traversal
910 _verify_cset_none, // no cset references before traversal
911 _verify_liveness_disable, // no reliable liveness data anymore
912 _verify_regions_notrash_nocset, // no trash and no cset regions
913 _verify_gcstate_stable // nothing forwarded before traversal
914 );
915 }
916
917 void ShenandoahVerifier::verify_after_traversal() {
918 verify_at_safepoint(
919 "After Traversal",
920 _verify_forwarded_none, // cannot have forwarded objects
921 _verify_marked_complete, // should have complete marking after traversal
922 _verify_cset_none, // no cset references left after traversal
923 _verify_liveness_disable, // liveness data is not collected for new allocations
924 _verify_regions_nocset, // no cset regions, trash regions allowed
925 _verify_gcstate_stable // nothing forwarded after traversal
926 );
927 }
928
929 void ShenandoahVerifier::verify_before_fullgc() {
930 verify_at_safepoint(
931 "Before Full GC",
932 _verify_forwarded_allow, // can have forwarded objects
933 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
934 _verify_cset_disable, // cset might be foobared
935 _verify_liveness_disable, // no reliable liveness data anymore
936 _verify_regions_disable, // no reliable region data here
937 _verify_gcstate_disable // no reliable gcstate data
938 );
939 }
940
941 void ShenandoahVerifier::verify_after_fullgc() {
942 verify_at_safepoint(
943 "After Full GC",
944 _verify_forwarded_none, // all objects are non-forwarded
945 _verify_marked_complete, // all objects are marked in complete bitmap
946 _verify_cset_none, // no cset references
947 _verify_liveness_disable, // no reliable liveness data anymore
948 _verify_regions_notrash_nocset, // no trash, no cset
949 _verify_gcstate_stable // full gc cleaned up everything
950 );
951 }
952
953 class ShenandoahVerifyNoForwared : public OopClosure {
954 private:
955 template <class T>
956 void do_oop_work(T* p) {
957 T o = RawAccess<>::oop_load(p);
958 if (!CompressedOops::is_null(o)) {
959 oop obj = CompressedOops::decode_not_null(o);
960 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
961 if (obj != fwd) {
962 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
963 "Verify Roots", "Should not be forwarded", __FILE__, __LINE__);
964 }
965 }
966 }
967
968 public:
969 void do_oop(narrowOop* p) { do_oop_work(p); }
970 void do_oop(oop* p) { do_oop_work(p); }
971 };
972
973 class ShenandoahVerifyInToSpaceClosure : public OopClosure {
974 private:
975 template <class T>
976 void do_oop_work(T* p) {
977 T o = RawAccess<>::oop_load(p);
978 if (!CompressedOops::is_null(o)) {
979 oop obj = CompressedOops::decode_not_null(o);
980 ShenandoahHeap* heap = ShenandoahHeap::heap_no_check();
981
982 if (!heap->marking_context()->is_marked(obj)) {
983 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
984 "Verify Roots In To-Space", "Should be marked", __FILE__, __LINE__);
985 }
986
987 if (heap->in_collection_set(obj)) {
988 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
989 "Verify Roots In To-Space", "Should not be in collection set", __FILE__, __LINE__);
990 }
991
992 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
993 if (obj != fwd) {
994 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
995 "Verify Roots In To-Space", "Should not be forwarded", __FILE__, __LINE__);
996 }
997 }
998 }
999
1000 public:
1001 void do_oop(narrowOop* p) { do_oop_work(p); }
1002 void do_oop(oop* p) { do_oop_work(p); }
1003 };
1004
1005 void ShenandoahVerifier::verify_roots_in_to_space() {
1006 ShenandoahRootVerifier verifier;
1007 ShenandoahVerifyInToSpaceClosure cl;
1008 verifier.oops_do(&cl);
1009 }
1010
1011 void ShenandoahVerifier::verify_roots_no_forwarded() {
1012 ShenandoahRootVerifier verifier;
1013 ShenandoahVerifyNoForwared cl;
1014 verifier.oops_do(&cl);
1015 }
1016
1017 void ShenandoahVerifier::verify_roots_no_forwarded_except(ShenandoahRootVerifier::RootTypes types) {
1018 ShenandoahRootVerifier verifier;
1019 verifier.excludes(types);
1020 ShenandoahVerifyNoForwared cl;
1021 verifier.oops_do(&cl);
1022 }