1 /*
2 * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shenandoah/shenandoahAsserts.hpp"
27 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
28 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
31 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
32 #include "gc/shenandoah/shenandoahUtils.hpp"
33 #include "gc/shenandoah/shenandoahVerifier.hpp"
34 #include "memory/allocation.hpp"
35 #include "memory/iterator.inline.hpp"
36 #include "memory/resourceArea.hpp"
37 #include "oops/compressedOops.inline.hpp"
38 #include "runtime/atomic.hpp"
39 #include "runtime/orderAccess.hpp"
40 #include "utilities/align.hpp"
41
42 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
43 #ifdef verify_oop
44 #undef verify_oop
45 #endif
46
47 class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
48 private:
49 const char* _phase;
50 ShenandoahVerifier::VerifyOptions _options;
51 ShenandoahVerifierStack* _stack;
52 ShenandoahHeap* _heap;
53 MarkBitMap* _map;
54 ShenandoahLivenessData* _ld;
55 void* _interior_loc;
56 oop _loc;
57
58 public:
59 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
60 const char* phase, ShenandoahVerifier::VerifyOptions options) :
61 _phase(phase),
62 _options(options),
63 _stack(stack),
64 _heap(ShenandoahHeap::heap()),
65 _map(map),
66 _ld(ld),
67 _interior_loc(NULL),
68 _loc(NULL) { }
69
70 private:
71 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
72 if (!test) {
73 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
74 }
75 }
76
77 template <class T>
78 void do_oop_work(T* p) {
79 T o = RawAccess<>::oop_load(p);
80 if (!CompressedOops::is_null(o)) {
81 oop obj = CompressedOops::decode_not_null(o);
82
83 // Single threaded verification can use faster non-atomic stack and bitmap
84 // methods.
85 //
86 // For performance reasons, only fully verify non-marked field values.
87 // We are here when the host object for *p is already marked.
88
89 HeapWord* addr = (HeapWord*) obj;
90 if (_map->par_mark(addr)) {
91 verify_oop_at(p, obj);
92 _stack->push(ShenandoahVerifierTask(obj));
93 }
94 }
95 }
96
97 void verify_oop(oop obj) {
98 // Perform consistency checks with gradually decreasing safety level. This guarantees
99 // that failure report would not try to touch something that was not yet verified to be
100 // safe to process.
101
102 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
103 "oop must be in heap");
104 check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
105 "oop must be aligned");
106
107 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
108 Klass* obj_klass = obj->klass_or_null();
109
110 // Verify that obj is not in dead space:
111 {
112 // Do this before touching obj->size()
113 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL,
114 "Object klass pointer should not be NULL");
115 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
116 "Object klass pointer must go to metaspace");
117
118 HeapWord *obj_addr = (HeapWord *) obj;
119 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
120 "Object start should be within the region");
121
122 if (!obj_reg->is_humongous()) {
123 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
124 "Object end should be within the region");
125 } else {
126 size_t humongous_start = obj_reg->region_number();
127 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
128 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
129 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
130 "Humongous object is in continuation that fits it");
131 }
132 }
133
134 // ------------ obj is safe at this point --------------
135
136 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
137 "Object should be in active region");
138
139 switch (_options._verify_liveness) {
140 case ShenandoahVerifier::_verify_liveness_disable:
141 // skip
142 break;
143 case ShenandoahVerifier::_verify_liveness_complete:
144 Atomic::add(&_ld[obj_reg->region_number()], (uint) obj->size());
145 // fallthrough for fast failure for un-live regions:
146 case ShenandoahVerifier::_verify_liveness_conservative:
147 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
148 "Object must belong to region with live data");
149 break;
150 default:
151 assert(false, "Unhandled liveness verification");
152 }
153 }
154
155 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
156
157 ShenandoahHeapRegion* fwd_reg = NULL;
158
159 if (obj != fwd) {
160 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
161 "Forwardee must be in heap");
162 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
163 "Forwardee is set");
164 check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
165 "Forwardee must be aligned");
166
167 // Do this before touching fwd->size()
168 Klass* fwd_klass = fwd->klass_or_null();
169 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL,
170 "Forwardee klass pointer should not be NULL");
171 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
172 "Forwardee klass pointer must go to metaspace");
173 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
174 "Forwardee klass pointer must go to metaspace");
175
176 fwd_reg = _heap->heap_region_containing(fwd);
177
178 // Verify that forwardee is not in the dead space:
179 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
180 "Should have no humongous forwardees");
181
182 HeapWord *fwd_addr = (HeapWord *) fwd;
183 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
184 "Forwardee start should be within the region");
185 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
186 "Forwardee end should be within the region");
187
188 oop fwd2 = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
189 check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
190 "Double forwarding");
191 } else {
192 fwd_reg = obj_reg;
193 }
194
195 // ------------ obj and fwd are safe at this point --------------
196
197 switch (_options._verify_marked) {
198 case ShenandoahVerifier::_verify_marked_disable:
199 // skip
200 break;
201 case ShenandoahVerifier::_verify_marked_incomplete:
202 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
203 "Must be marked in incomplete bitmap");
204 break;
205 case ShenandoahVerifier::_verify_marked_complete:
206 check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj),
207 "Must be marked in complete bitmap");
208 break;
209 default:
210 assert(false, "Unhandled mark verification");
211 }
212
213 switch (_options._verify_forwarded) {
214 case ShenandoahVerifier::_verify_forwarded_disable:
215 // skip
216 break;
217 case ShenandoahVerifier::_verify_forwarded_none: {
218 check(ShenandoahAsserts::_safe_all, obj, (obj == fwd),
219 "Should not be forwarded");
220 break;
221 }
222 case ShenandoahVerifier::_verify_forwarded_allow: {
223 if (obj != fwd) {
224 check(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg,
225 "Forwardee should be in another region");
226 }
227 break;
228 }
229 default:
230 assert(false, "Unhandled forwarding verification");
231 }
232
233 switch (_options._verify_cset) {
234 case ShenandoahVerifier::_verify_cset_disable:
235 // skip
236 break;
237 case ShenandoahVerifier::_verify_cset_none:
238 check(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj),
239 "Should not have references to collection set");
240 break;
241 case ShenandoahVerifier::_verify_cset_forwarded:
242 if (_heap->in_collection_set(obj)) {
243 check(ShenandoahAsserts::_safe_all, obj, (obj != fwd),
244 "Object in collection set, should have forwardee");
245 }
246 break;
247 default:
248 assert(false, "Unhandled cset verification");
249 }
250
251 }
252
253 public:
254 /**
255 * Verify object with known interior reference.
256 * @param p interior reference where the object is referenced from; can be off-heap
257 * @param obj verified object
258 */
259 template <class T>
260 void verify_oop_at(T* p, oop obj) {
261 _interior_loc = p;
262 verify_oop(obj);
263 _interior_loc = NULL;
264 }
265
266 /**
267 * Verify object without known interior reference.
268 * Useful when picking up the object at known offset in heap,
269 * but without knowing what objects reference it.
270 * @param obj verified object
271 */
272 void verify_oop_standalone(oop obj) {
273 _interior_loc = NULL;
274 verify_oop(obj);
275 _interior_loc = NULL;
276 }
277
278 /**
279 * Verify oop fields from this object.
280 * @param obj host object for verified fields
281 */
282 void verify_oops_from(oop obj) {
283 _loc = obj;
284 obj->oop_iterate(this);
285 _loc = NULL;
286 }
287
288 virtual void do_oop(oop* p) { do_oop_work(p); }
289 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
290 };
291
292 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
293 private:
294 size_t _used, _committed, _garbage;
295 public:
296 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
297
298 void heap_region_do(ShenandoahHeapRegion* r) {
299 _used += r->used();
300 _garbage += r->garbage();
301 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
302 }
303
304 size_t used() { return _used; }
305 size_t committed() { return _committed; }
306 size_t garbage() { return _garbage; }
307 };
308
309 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure {
310 private:
311 ShenandoahHeap* _heap;
312 const char* _phase;
313 ShenandoahVerifier::VerifyRegions _regions;
314 public:
315 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) :
316 _heap(ShenandoahHeap::heap()),
317 _phase(phase),
318 _regions(regions) {};
319
320 void print_failure(ShenandoahHeapRegion* r, const char* label) {
321 ResourceMark rm;
322
323 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label);
324
325 stringStream ss;
326 r->print_on(&ss);
327 msg.append("%s", ss.as_string());
328
329 report_vm_error(__FILE__, __LINE__, msg.buffer());
330 }
331
332 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) {
333 if (!test) {
334 print_failure(r, msg);
335 }
336 }
337
338 void heap_region_do(ShenandoahHeapRegion* r) {
339 switch (_regions) {
340 case ShenandoahVerifier::_verify_regions_disable:
341 break;
342 case ShenandoahVerifier::_verify_regions_notrash:
343 verify(r, !r->is_trash(),
344 "Should not have trash regions");
345 break;
346 case ShenandoahVerifier::_verify_regions_nocset:
347 verify(r, !r->is_cset(),
348 "Should not have cset regions");
349 break;
350 case ShenandoahVerifier::_verify_regions_notrash_nocset:
351 verify(r, !r->is_trash(),
352 "Should not have trash regions");
353 verify(r, !r->is_cset(),
354 "Should not have cset regions");
355 break;
356 default:
357 ShouldNotReachHere();
358 }
359
360 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
361 "Capacity should match region size");
362
363 verify(r, r->bottom() <= r->top(),
364 "Region top should not be less than bottom");
365
366 verify(r, r->bottom() <= _heap->marking_context()->top_at_mark_start(r),
367 "Region TAMS should not be less than bottom");
368
369 verify(r, _heap->marking_context()->top_at_mark_start(r) <= r->top(),
370 "Complete TAMS should not be larger than top");
371
372 verify(r, r->get_live_data_bytes() <= r->capacity(),
373 "Live data cannot be larger than capacity");
374
375 verify(r, r->garbage() <= r->capacity(),
376 "Garbage cannot be larger than capacity");
377
378 verify(r, r->used() <= r->capacity(),
379 "Used cannot be larger than capacity");
380
381 verify(r, r->get_shared_allocs() <= r->capacity(),
382 "Shared alloc count should not be larger than capacity");
383
384 verify(r, r->get_tlab_allocs() <= r->capacity(),
385 "TLAB alloc count should not be larger than capacity");
386
387 verify(r, r->get_gclab_allocs() <= r->capacity(),
388 "GCLAB alloc count should not be larger than capacity");
389
390 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
391 "Accurate accounting: shared + TLAB + GCLAB = used");
392
393 verify(r, !r->is_empty() || !r->has_live(),
394 "Empty regions should not have live data");
395
396 verify(r, r->is_cset() == _heap->collection_set()->is_in(r),
397 "Transitional: region flags and collection set agree");
398
399 verify(r, r->is_empty() || r->seqnum_first_alloc() != 0,
400 "Non-empty regions should have first seqnum set");
401
402 verify(r, r->is_empty() || (r->seqnum_first_alloc_mutator() != 0 || r->seqnum_first_alloc_gc() != 0),
403 "Non-empty regions should have first seqnum set to either GC or mutator");
404
405 verify(r, r->is_empty() || r->seqnum_last_alloc() != 0,
406 "Non-empty regions should have last seqnum set");
407
408 verify(r, r->is_empty() || (r->seqnum_last_alloc_mutator() != 0 || r->seqnum_last_alloc_gc() != 0),
409 "Non-empty regions should have last seqnum set to either GC or mutator");
410
411 verify(r, r->seqnum_first_alloc() <= r->seqnum_last_alloc(),
412 "First seqnum should not be greater than last timestamp");
413
414 verify(r, r->seqnum_first_alloc_mutator() <= r->seqnum_last_alloc_mutator(),
415 "First mutator seqnum should not be greater than last seqnum");
416
417 verify(r, r->seqnum_first_alloc_gc() <= r->seqnum_last_alloc_gc(),
418 "First GC seqnum should not be greater than last seqnum");
419 }
420 };
421
422 class ShenandoahVerifierReachableTask : public AbstractGangTask {
423 private:
424 const char* _label;
425 ShenandoahRootVerifier* _verifier;
426 ShenandoahVerifier::VerifyOptions _options;
427 ShenandoahHeap* _heap;
428 ShenandoahLivenessData* _ld;
429 MarkBitMap* _bitmap;
430 volatile size_t _processed;
431
432 public:
433 ShenandoahVerifierReachableTask(MarkBitMap* bitmap,
434 ShenandoahLivenessData* ld,
435 ShenandoahRootVerifier* verifier,
436 const char* label,
437 ShenandoahVerifier::VerifyOptions options) :
438 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"),
439 _label(label),
440 _verifier(verifier),
441 _options(options),
442 _heap(ShenandoahHeap::heap()),
443 _ld(ld),
444 _bitmap(bitmap),
445 _processed(0) {};
446
447 size_t processed() {
448 return _processed;
449 }
450
451 virtual void work(uint worker_id) {
452 ResourceMark rm;
453 ShenandoahVerifierStack stack;
454
455 // On level 2, we need to only check the roots once.
456 // On level 3, we want to check the roots, and seed the local stack.
457 // It is a lesser evil to accept multiple root scans at level 3, because
458 // extended parallelism would buy us out.
459 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0))
460 || (ShenandoahVerifyLevel >= 3)) {
461 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
462 ShenandoahMessageBuffer("%s, Roots", _label),
463 _options);
464 if (_heap->unload_classes()) {
465 _verifier->strong_roots_do(&cl);
466 } else {
467 _verifier->roots_do(&cl);
468 }
469 }
470
471 size_t processed = 0;
472
473 if (ShenandoahVerifyLevel >= 3) {
474 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
475 ShenandoahMessageBuffer("%s, Reachable", _label),
476 _options);
477 while (!stack.is_empty()) {
478 processed++;
479 ShenandoahVerifierTask task = stack.pop();
480 cl.verify_oops_from(task.obj());
481 }
482 }
483
484 Atomic::add(&_processed, processed);
485 }
486 };
487
488 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask {
489 private:
490 const char* _label;
491 ShenandoahVerifier::VerifyOptions _options;
492 ShenandoahHeap *_heap;
493 MarkBitMap* _bitmap;
494 ShenandoahLivenessData* _ld;
495 volatile size_t _claimed;
496 volatile size_t _processed;
497
498 public:
499 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
500 ShenandoahLivenessData* ld,
501 const char* label,
502 ShenandoahVerifier::VerifyOptions options) :
503 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"),
504 _label(label),
505 _options(options),
506 _heap(ShenandoahHeap::heap()),
507 _bitmap(bitmap),
508 _ld(ld),
509 _claimed(0),
510 _processed(0) {};
511
512 size_t processed() {
513 return _processed;
514 }
515
516 virtual void work(uint worker_id) {
517 ShenandoahVerifierStack stack;
518 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
519 ShenandoahMessageBuffer("%s, Marked", _label),
520 _options);
521
522 while (true) {
523 size_t v = Atomic::add(&_claimed, 1u) - 1;
524 if (v < _heap->num_regions()) {
525 ShenandoahHeapRegion* r = _heap->get_region(v);
526 if (!r->is_humongous() && !r->is_trash()) {
527 work_regular(r, stack, cl);
528 } else if (r->is_humongous_start()) {
529 work_humongous(r, stack, cl);
530 }
531 } else {
532 break;
533 }
534 }
535 }
536
537 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
538 size_t processed = 0;
539 HeapWord* obj = r->bottom();
540 if (_heap->complete_marking_context()->is_marked((oop)obj)) {
541 verify_and_follow(obj, stack, cl, &processed);
542 }
543 Atomic::add(&_processed, processed);
544 }
545
546 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
547 size_t processed = 0;
548 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
549 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
550
551 // Bitmaps, before TAMS
552 if (tams > r->bottom()) {
553 HeapWord* start = r->bottom();
554 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
555
556 while (addr < tams) {
557 verify_and_follow(addr, stack, cl, &processed);
558 addr += 1;
559 if (addr < tams) {
560 addr = mark_bit_map->get_next_marked_addr(addr, tams);
561 }
562 }
563 }
564
565 // Size-based, after TAMS
566 {
567 HeapWord* limit = r->top();
568 HeapWord* addr = tams;
569
570 while (addr < limit) {
571 verify_and_follow(addr, stack, cl, &processed);
572 addr += oop(addr)->size();
573 }
574 }
575
576 Atomic::add(&_processed, processed);
577 }
578
579 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
580 if (!_bitmap->par_mark(addr)) return;
581
582 // Verify the object itself:
583 oop obj = oop(addr);
584 cl.verify_oop_standalone(obj);
585
586 // Verify everything reachable from that object too, hopefully realizing
587 // everything was already marked, and never touching further:
588 cl.verify_oops_from(obj);
589 (*processed)++;
590
591 while (!stack.is_empty()) {
592 ShenandoahVerifierTask task = stack.pop();
593 cl.verify_oops_from(task.obj());
594 (*processed)++;
595 }
596 }
597 };
598
599 class VerifyThreadGCState : public ThreadClosure {
600 private:
601 const char* _label;
602 char _expected;
603
604 public:
605 VerifyThreadGCState(const char* label, char expected) : _expected(expected) {}
606 void do_thread(Thread* t) {
607 char actual = ShenandoahThreadLocalData::gc_state(t);
608 if (actual != _expected) {
609 fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual);
610 }
611 }
612 };
613
614 class ShenandoahGCStateResetter : public StackObj {
615 private:
616 ShenandoahHeap* const _heap;
617 char _gc_state;
618
619 public:
620 ShenandoahGCStateResetter() : _heap(ShenandoahHeap::heap()) {
621 _gc_state = _heap->gc_state();
622 _heap->_gc_state.clear();
623 }
624
625 ~ShenandoahGCStateResetter() {
626 _heap->_gc_state.set(_gc_state);
627 assert(_heap->gc_state() == _gc_state, "Should be restored");
628 }
629 };
630
631 void ShenandoahVerifier::verify_at_safepoint(const char *label,
632 VerifyForwarded forwarded, VerifyMarked marked,
633 VerifyCollectionSet cset,
634 VerifyLiveness liveness, VerifyRegions regions,
635 VerifyGCState gcstate) {
636 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
637 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
638
639 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
640 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
641
642 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel);
643
644 // GC state checks
645 {
646 char expected = -1;
647 bool enabled;
648 switch (gcstate) {
649 case _verify_gcstate_disable:
650 enabled = false;
651 break;
652 case _verify_gcstate_forwarded:
653 enabled = true;
654 expected = ShenandoahHeap::HAS_FORWARDED;
655 break;
656 case _verify_gcstate_evacuation:
657 enabled = true;
658 expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION;
659 break;
660 case _verify_gcstate_stable:
661 enabled = true;
662 expected = ShenandoahHeap::STABLE;
663 break;
664 default:
665 enabled = false;
666 assert(false, "Unhandled gc-state verification");
667 }
668
669 if (enabled) {
670 char actual = _heap->gc_state();
671 if (actual != expected) {
672 fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual);
673 }
674
675 VerifyThreadGCState vtgcs(label, expected);
676 Threads::java_threads_do(&vtgcs);
677 }
678 }
679
680 // Deactivate barriers temporarily: Verifier wants plain heap accesses
681 ShenandoahGCStateResetter resetter;
682
683 // Heap size checks
684 {
685 ShenandoahHeapLocker lock(_heap->lock());
686
687 ShenandoahCalculateRegionStatsClosure cl;
688 _heap->heap_region_iterate(&cl);
689 size_t heap_used = _heap->used();
690 guarantee(cl.used() == heap_used,
691 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s",
692 label,
693 byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used),
694 byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used()));
695
696 size_t heap_committed = _heap->committed();
697 guarantee(cl.committed() == heap_committed,
698 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "%s, regions-committed = " SIZE_FORMAT "%s",
699 label,
700 byte_size_in_proper_unit(heap_committed), proper_unit_for_byte_size(heap_committed),
701 byte_size_in_proper_unit(cl.committed()), proper_unit_for_byte_size(cl.committed()));
702 }
703
704 // Internal heap region checks
705 if (ShenandoahVerifyLevel >= 1) {
706 ShenandoahVerifyHeapRegionClosure cl(label, regions);
707 _heap->heap_region_iterate(&cl);
708 }
709
710 OrderAccess::fence();
711 _heap->make_parsable(false);
712
713 // Allocate temporary bitmap for storing marking wavefront:
714 _verification_bit_map->clear();
715
716 // Allocate temporary array for storing liveness data
717 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC);
718 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0);
719
720 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, cset, liveness, regions, gcstate);
721
722 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
723 // This verifies what application can see, since it only cares about reachable objects.
724 size_t count_reachable = 0;
725 if (ShenandoahVerifyLevel >= 2) {
726 ShenandoahRootVerifier verifier;
727
728 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &verifier, label, options);
729 _heap->workers()->run_task(&task);
730 count_reachable = task.processed();
731 }
732
733 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
734 // not the application, can see during the region scans. There is no reason to process the objects
735 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
736 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
737 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
738 // version
739
740 size_t count_marked = 0;
741 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
742 guarantee(_heap->marking_context()->is_complete(), "Marking context should be complete");
743 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options);
744 _heap->workers()->run_task(&task);
745 count_marked = task.processed();
746 } else {
747 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_incomplete || marked == _verify_marked_disable, "Should be");
748 }
749
750 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
751 // marked objects.
752
753 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
754 for (size_t i = 0; i < _heap->num_regions(); i++) {
755 ShenandoahHeapRegion* r = _heap->get_region(i);
756
757 juint verf_live = 0;
758 if (r->is_humongous()) {
759 // For humongous objects, test if start region is marked live, and if so,
760 // all humongous regions in that chain have live data equal to their "used".
761 juint start_live = Atomic::load_acquire(&ld[r->humongous_start_region()->region_number()]);
762 if (start_live > 0) {
763 verf_live = (juint)(r->used() / HeapWordSize);
764 }
765 } else {
766 verf_live = Atomic::load_acquire(&ld[r->region_number()]);
767 }
768
769 size_t reg_live = r->get_live_data_words();
770 if (reg_live != verf_live) {
771 ResourceMark rm;
772 stringStream ss;
773 r->print_on(&ss);
774 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s",
775 label, reg_live, verf_live, ss.as_string());
776 }
777 }
778 }
779
780 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)",
781 label, ShenandoahVerifyLevel, count_reachable, count_marked);
782
783 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld);
784 }
785
786 void ShenandoahVerifier::verify_generic(VerifyOption vo) {
787 verify_at_safepoint(
788 "Generic Verification",
789 _verify_forwarded_allow, // conservatively allow forwarded
790 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
791 _verify_cset_disable, // cset may be inconsistent
792 _verify_liveness_disable, // no reliable liveness data
793 _verify_regions_disable, // no reliable region data
794 _verify_gcstate_disable // no data about gcstate
795 );
796 }
797
798 void ShenandoahVerifier::verify_before_concmark() {
799 if (_heap->has_forwarded_objects()) {
800 verify_at_safepoint(
801 "Before Mark",
802 _verify_forwarded_allow, // may have forwarded references
803 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
804 _verify_cset_forwarded, // allow forwarded references to cset
805 _verify_liveness_disable, // no reliable liveness data
806 _verify_regions_notrash, // no trash regions
807 _verify_gcstate_forwarded // there are forwarded objects
808 );
809 } else {
810 verify_at_safepoint(
811 "Before Mark",
812 _verify_forwarded_none, // UR should have fixed up
813 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
814 _verify_cset_none, // UR should have fixed this
815 _verify_liveness_disable, // no reliable liveness data
816 _verify_regions_notrash, // no trash regions
817 _verify_gcstate_stable // there are no forwarded objects
818 );
819 }
820 }
821
822 void ShenandoahVerifier::verify_after_concmark() {
823 verify_at_safepoint(
824 "After Mark",
825 _verify_forwarded_none, // no forwarded references
826 _verify_marked_complete, // bitmaps as precise as we can get
827 _verify_cset_none, // no references to cset anymore
828 _verify_liveness_complete, // liveness data must be complete here
829 _verify_regions_disable, // trash regions not yet recycled
830 _verify_gcstate_stable // mark should have stabilized the heap
831 );
832 }
833
834 void ShenandoahVerifier::verify_before_evacuation() {
835 verify_at_safepoint(
836 "Before Evacuation",
837 _verify_forwarded_none, // no forwarded references
838 _verify_marked_complete, // walk over marked objects too
839 _verify_cset_disable, // non-forwarded references to cset expected
840 _verify_liveness_complete, // liveness data must be complete here
841 _verify_regions_disable, // trash regions not yet recycled
842 _verify_gcstate_stable // mark should have stabilized the heap
843 );
844 }
845
846 void ShenandoahVerifier::verify_during_evacuation() {
847 verify_at_safepoint(
848 "During Evacuation",
849 _verify_forwarded_allow, // some forwarded references are allowed
850 _verify_marked_disable, // walk only roots
851 _verify_cset_disable, // some cset references are not forwarded yet
852 _verify_liveness_disable, // liveness data might be already stale after pre-evacs
853 _verify_regions_disable, // trash regions not yet recycled
854 _verify_gcstate_evacuation // evacuation is in progress
855 );
856 }
857
858 void ShenandoahVerifier::verify_after_evacuation() {
859 verify_at_safepoint(
860 "After Evacuation",
861 _verify_forwarded_allow, // objects are still forwarded
862 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
863 _verify_cset_forwarded, // all cset refs are fully forwarded
864 _verify_liveness_disable, // no reliable liveness data anymore
865 _verify_regions_notrash, // trash regions have been recycled already
866 _verify_gcstate_forwarded // evacuation produced some forwarded objects
867 );
868 }
869
870 void ShenandoahVerifier::verify_before_updaterefs() {
871 verify_at_safepoint(
872 "Before Updating References",
873 _verify_forwarded_allow, // forwarded references allowed
874 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
875 _verify_cset_forwarded, // all cset refs are fully forwarded
876 _verify_liveness_disable, // no reliable liveness data anymore
877 _verify_regions_notrash, // trash regions have been recycled already
878 _verify_gcstate_forwarded // evacuation should have produced some forwarded objects
879 );
880 }
881
882 void ShenandoahVerifier::verify_after_updaterefs() {
883 verify_at_safepoint(
884 "After Updating References",
885 _verify_forwarded_none, // no forwarded references
886 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
887 _verify_cset_none, // no cset references, all updated
888 _verify_liveness_disable, // no reliable liveness data anymore
889 _verify_regions_nocset, // no cset regions, trash regions have appeared
890 _verify_gcstate_stable // update refs had cleaned up forwarded objects
891 );
892 }
893
894 void ShenandoahVerifier::verify_after_degenerated() {
895 verify_at_safepoint(
896 "After Degenerated GC",
897 _verify_forwarded_none, // all objects are non-forwarded
898 _verify_marked_complete, // all objects are marked in complete bitmap
899 _verify_cset_none, // no cset references
900 _verify_liveness_disable, // no reliable liveness data anymore
901 _verify_regions_notrash_nocset, // no trash, no cset
902 _verify_gcstate_stable // degenerated refs had cleaned up forwarded objects
903 );
904 }
905
906 void ShenandoahVerifier::verify_before_traversal() {
907 verify_at_safepoint(
908 "Before Traversal",
909 _verify_forwarded_none, // cannot have forwarded objects
910 _verify_marked_disable, // bitmaps are not relevant before traversal
911 _verify_cset_none, // no cset references before traversal
912 _verify_liveness_disable, // no reliable liveness data anymore
913 _verify_regions_notrash_nocset, // no trash and no cset regions
914 _verify_gcstate_stable // nothing forwarded before traversal
915 );
916 }
917
918 void ShenandoahVerifier::verify_after_traversal() {
919 verify_at_safepoint(
920 "After Traversal",
921 _verify_forwarded_none, // cannot have forwarded objects
922 _verify_marked_complete, // should have complete marking after traversal
923 _verify_cset_none, // no cset references left after traversal
924 _verify_liveness_disable, // liveness data is not collected for new allocations
925 _verify_regions_nocset, // no cset regions, trash regions allowed
926 _verify_gcstate_stable // nothing forwarded after traversal
927 );
928 }
929
930 void ShenandoahVerifier::verify_before_fullgc() {
931 verify_at_safepoint(
932 "Before Full GC",
933 _verify_forwarded_allow, // can have forwarded objects
934 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations
935 _verify_cset_disable, // cset might be foobared
936 _verify_liveness_disable, // no reliable liveness data anymore
937 _verify_regions_disable, // no reliable region data here
938 _verify_gcstate_disable // no reliable gcstate data
939 );
940 }
941
942 void ShenandoahVerifier::verify_after_fullgc() {
943 verify_at_safepoint(
944 "After Full GC",
945 _verify_forwarded_none, // all objects are non-forwarded
946 _verify_marked_complete, // all objects are marked in complete bitmap
947 _verify_cset_none, // no cset references
948 _verify_liveness_disable, // no reliable liveness data anymore
949 _verify_regions_notrash_nocset, // no trash, no cset
950 _verify_gcstate_stable // full gc cleaned up everything
951 );
952 }
953
954 class ShenandoahVerifyNoForwared : public OopClosure {
955 private:
956 template <class T>
957 void do_oop_work(T* p) {
958 T o = RawAccess<>::oop_load(p);
959 if (!CompressedOops::is_null(o)) {
960 oop obj = CompressedOops::decode_not_null(o);
961 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
962 if (obj != fwd) {
963 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
964 "Verify Roots", "Should not be forwarded", __FILE__, __LINE__);
965 }
966 }
967 }
968
969 public:
970 void do_oop(narrowOop* p) { do_oop_work(p); }
971 void do_oop(oop* p) { do_oop_work(p); }
972 };
973
974 class ShenandoahVerifyInToSpaceClosure : public OopClosure {
975 private:
976 template <class T>
977 void do_oop_work(T* p) {
978 T o = RawAccess<>::oop_load(p);
979 if (!CompressedOops::is_null(o)) {
980 oop obj = CompressedOops::decode_not_null(o);
981 ShenandoahHeap* heap = ShenandoahHeap::heap_no_check();
982
983 if (!heap->marking_context()->is_marked(obj)) {
984 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
985 "Verify Roots In To-Space", "Should be marked", __FILE__, __LINE__);
986 }
987
988 if (heap->in_collection_set(obj)) {
989 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
990 "Verify Roots In To-Space", "Should not be in collection set", __FILE__, __LINE__);
991 }
992
993 oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
994 if (obj != fwd) {
995 ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
996 "Verify Roots In To-Space", "Should not be forwarded", __FILE__, __LINE__);
997 }
998 }
999 }
1000
1001 public:
1002 void do_oop(narrowOop* p) { do_oop_work(p); }
1003 void do_oop(oop* p) { do_oop_work(p); }
1004 };
1005
1006 void ShenandoahVerifier::verify_roots_in_to_space() {
1007 ShenandoahRootVerifier verifier;
1008 ShenandoahVerifyInToSpaceClosure cl;
1009 verifier.oops_do(&cl);
1010 }
1011
1012 void ShenandoahVerifier::verify_roots_no_forwarded() {
1013 ShenandoahRootVerifier verifier;
1014 ShenandoahVerifyNoForwared cl;
1015 verifier.oops_do(&cl);
1016 }
1017
1018 void ShenandoahVerifier::verify_roots_no_forwarded_except(ShenandoahRootVerifier::RootTypes types) {
1019 ShenandoahRootVerifier verifier;
1020 verifier.excludes(types);
1021 ShenandoahVerifyNoForwared cl;
1022 verifier.oops_do(&cl);
1023 }