1 /*
   2  * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "gc/shenandoah/shenandoahAsserts.hpp"
  27 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
  28 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  30 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
  31 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  32 #include "gc/shenandoah/shenandoahUtils.hpp"
  33 #include "gc/shenandoah/shenandoahVerifier.hpp"
  34 #include "memory/allocation.hpp"
  35 #include "memory/iterator.inline.hpp"
  36 #include "memory/resourceArea.hpp"
  37 #include "oops/compressedOops.inline.hpp"
  38 #include "utilities/align.hpp"
  39 
  40 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp)
  41 #ifdef verify_oop
  42 #undef verify_oop
  43 #endif
  44 
  45 class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
  46 private:
  47   const char* _phase;
  48   ShenandoahVerifier::VerifyOptions _options;
  49   ShenandoahVerifierStack* _stack;
  50   ShenandoahHeap* _heap;
  51   MarkBitMap* _map;
  52   ShenandoahLivenessData* _ld;
  53   void* _interior_loc;
  54   oop _loc;
  55 
  56 public:
  57   ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld,
  58                              const char* phase, ShenandoahVerifier::VerifyOptions options) :
  59     _phase(phase),
  60     _options(options),
  61     _stack(stack),
  62     _heap(ShenandoahHeap::heap()),
  63     _map(map),
  64     _ld(ld),
  65     _interior_loc(NULL),
  66     _loc(NULL) { }
  67 
  68 private:
  69   void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
  70     if (!test) {
  71       ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__);
  72     }
  73   }
  74 
  75   template <class T>
  76   void do_oop_work(T* p) {
  77     T o = RawAccess<>::oop_load(p);
  78     if (!CompressedOops::is_null(o)) {
  79       oop obj = CompressedOops::decode_not_null(o);
  80 
  81       // Single threaded verification can use faster non-atomic stack and bitmap
  82       // methods.
  83       //
  84       // For performance reasons, only fully verify non-marked field values.
  85       // We are here when the host object for *p is already marked.
  86 
  87       HeapWord* addr = (HeapWord*) obj;
  88       if (_map->par_mark(addr)) {
  89         verify_oop_at(p, obj);
  90         _stack->push(ShenandoahVerifierTask(obj));
  91       }
  92     }
  93   }
  94 
  95   void verify_oop(oop obj) {
  96     // Perform consistency checks with gradually decreasing safety level. This guarantees
  97     // that failure report would not try to touch something that was not yet verified to be
  98     // safe to process.
  99 
 100     check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj),
 101               "oop must be in heap");
 102     check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj),
 103               "oop must be aligned");
 104 
 105     ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
 106     Klass* obj_klass = obj->klass_or_null();
 107 
 108     // Verify that obj is not in dead space:
 109     {
 110       // Do this before touching obj->size()
 111       check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL,
 112              "Object klass pointer should not be NULL");
 113       check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass),
 114              "Object klass pointer must go to metaspace");
 115 
 116       HeapWord *obj_addr = (HeapWord *) obj;
 117       check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(),
 118              "Object start should be within the region");
 119 
 120       if (!obj_reg->is_humongous()) {
 121         check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
 122                "Object end should be within the region");
 123       } else {
 124         size_t humongous_start = obj_reg->region_number();
 125         size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
 126         for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
 127           check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
 128                  "Humongous object is in continuation that fits it");
 129         }
 130       }
 131 
 132       // ------------ obj is safe at this point --------------
 133 
 134       check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
 135             "Object should be in active region");
 136 
 137       switch (_options._verify_liveness) {
 138         case ShenandoahVerifier::_verify_liveness_disable:
 139           // skip
 140           break;
 141         case ShenandoahVerifier::_verify_liveness_complete:
 142           Atomic::add((uint) obj->size(), &_ld[obj_reg->region_number()]);
 143           // fallthrough for fast failure for un-live regions:
 144         case ShenandoahVerifier::_verify_liveness_conservative:
 145           check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
 146                    "Object must belong to region with live data");
 147           break;
 148         default:
 149           assert(false, "Unhandled liveness verification");
 150       }
 151     }
 152 
 153     oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
 154 
 155     ShenandoahHeapRegion* fwd_reg = NULL;
 156 
 157     if (obj != fwd) {
 158       check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd),
 159              "Forwardee must be in heap");
 160       check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd),
 161              "Forwardee is set");
 162       check(ShenandoahAsserts::_safe_oop, obj, is_object_aligned(fwd),
 163              "Forwardee must be aligned");
 164 
 165       // Do this before touching fwd->size()
 166       Klass* fwd_klass = fwd->klass_or_null();
 167       check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL,
 168              "Forwardee klass pointer should not be NULL");
 169       check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass),
 170              "Forwardee klass pointer must go to metaspace");
 171       check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass,
 172              "Forwardee klass pointer must go to metaspace");
 173 
 174       fwd_reg = _heap->heap_region_containing(fwd);
 175 
 176       // Verify that forwardee is not in the dead space:
 177       check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(),
 178              "Should have no humongous forwardees");
 179 
 180       HeapWord *fwd_addr = (HeapWord *) fwd;
 181       check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
 182              "Forwardee start should be within the region");
 183       check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
 184              "Forwardee end should be within the region");
 185 
 186       oop fwd2 = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
 187       check(ShenandoahAsserts::_safe_oop, obj, (fwd == fwd2),
 188              "Double forwarding");
 189     } else {
 190       fwd_reg = obj_reg;
 191     }
 192 
 193     // ------------ obj and fwd are safe at this point --------------
 194 
 195     switch (_options._verify_marked) {
 196       case ShenandoahVerifier::_verify_marked_disable:
 197         // skip
 198         break;
 199       case ShenandoahVerifier::_verify_marked_incomplete:
 200         check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj),
 201                "Must be marked in incomplete bitmap");
 202         break;
 203       case ShenandoahVerifier::_verify_marked_complete:
 204         check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj),
 205                "Must be marked in complete bitmap");
 206         break;
 207       default:
 208         assert(false, "Unhandled mark verification");
 209     }
 210 
 211     switch (_options._verify_forwarded) {
 212       case ShenandoahVerifier::_verify_forwarded_disable:
 213         // skip
 214         break;
 215       case ShenandoahVerifier::_verify_forwarded_none: {
 216         check(ShenandoahAsserts::_safe_all, obj, (obj == fwd),
 217                "Should not be forwarded");
 218         break;
 219       }
 220       case ShenandoahVerifier::_verify_forwarded_allow: {
 221         if (obj != fwd) {
 222           check(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg,
 223                  "Forwardee should be in another region");
 224         }
 225         break;
 226       }
 227       default:
 228         assert(false, "Unhandled forwarding verification");
 229     }
 230 
 231     switch (_options._verify_cset) {
 232       case ShenandoahVerifier::_verify_cset_disable:
 233         // skip
 234         break;
 235       case ShenandoahVerifier::_verify_cset_none:
 236         check(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj),
 237                "Should not have references to collection set");
 238         break;
 239       case ShenandoahVerifier::_verify_cset_forwarded:
 240         if (_heap->in_collection_set(obj)) {
 241           check(ShenandoahAsserts::_safe_all, obj, (obj != fwd),
 242                  "Object in collection set, should have forwardee");
 243         }
 244         break;
 245       default:
 246         assert(false, "Unhandled cset verification");
 247     }
 248 
 249   }
 250 
 251 public:
 252   /**
 253    * Verify object with known interior reference.
 254    * @param p interior reference where the object is referenced from; can be off-heap
 255    * @param obj verified object
 256    */
 257   template <class T>
 258   void verify_oop_at(T* p, oop obj) {
 259     _interior_loc = p;
 260     verify_oop(obj);
 261     _interior_loc = NULL;
 262   }
 263 
 264   /**
 265    * Verify object without known interior reference.
 266    * Useful when picking up the object at known offset in heap,
 267    * but without knowing what objects reference it.
 268    * @param obj verified object
 269    */
 270   void verify_oop_standalone(oop obj) {
 271     _interior_loc = NULL;
 272     verify_oop(obj);
 273     _interior_loc = NULL;
 274   }
 275 
 276   /**
 277    * Verify oop fields from this object.
 278    * @param obj host object for verified fields
 279    */
 280   void verify_oops_from(oop obj) {
 281     _loc = obj;
 282     obj->oop_iterate(this);
 283     _loc = NULL;
 284   }
 285 
 286   virtual void do_oop(oop* p) { do_oop_work(p); }
 287   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
 288 };
 289 
 290 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
 291 private:
 292   size_t _used, _committed, _garbage;
 293 public:
 294   ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {};
 295 
 296   void heap_region_do(ShenandoahHeapRegion* r) {
 297     _used += r->used();
 298     _garbage += r->garbage();
 299     _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
 300   }
 301 
 302   size_t used() { return _used; }
 303   size_t committed() { return _committed; }
 304   size_t garbage() { return _garbage; }
 305 };
 306 
 307 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure {
 308 private:
 309   ShenandoahHeap* _heap;
 310   const char* _phase;
 311   ShenandoahVerifier::VerifyRegions _regions;
 312 public:
 313   ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) :
 314     _heap(ShenandoahHeap::heap()),
 315     _phase(phase),
 316     _regions(regions) {};
 317 
 318   void print_failure(ShenandoahHeapRegion* r, const char* label) {
 319     ResourceMark rm;
 320 
 321     ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label);
 322 
 323     stringStream ss;
 324     r->print_on(&ss);
 325     msg.append("%s", ss.as_string());
 326 
 327     report_vm_error(__FILE__, __LINE__, msg.buffer());
 328   }
 329 
 330   void verify(ShenandoahHeapRegion* r, bool test, const char* msg) {
 331     if (!test) {
 332       print_failure(r, msg);
 333     }
 334   }
 335 
 336   void heap_region_do(ShenandoahHeapRegion* r) {
 337     switch (_regions) {
 338       case ShenandoahVerifier::_verify_regions_disable:
 339         break;
 340       case ShenandoahVerifier::_verify_regions_notrash:
 341         verify(r, !r->is_trash(),
 342                "Should not have trash regions");
 343         break;
 344       case ShenandoahVerifier::_verify_regions_nocset:
 345         verify(r, !r->is_cset(),
 346                "Should not have cset regions");
 347         break;
 348       case ShenandoahVerifier::_verify_regions_notrash_nocset:
 349         verify(r, !r->is_trash(),
 350                "Should not have trash regions");
 351         verify(r, !r->is_cset(),
 352                "Should not have cset regions");
 353         break;
 354       default:
 355         ShouldNotReachHere();
 356     }
 357 
 358     verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(),
 359            "Capacity should match region size");
 360 
 361     verify(r, r->bottom() <= r->top(),
 362            "Region top should not be less than bottom");
 363 
 364     verify(r, r->bottom() <= _heap->marking_context()->top_at_mark_start(r),
 365            "Region TAMS should not be less than bottom");
 366 
 367     verify(r, _heap->marking_context()->top_at_mark_start(r) <= r->top(),
 368            "Complete TAMS should not be larger than top");
 369 
 370     verify(r, r->get_live_data_bytes() <= r->capacity(),
 371            "Live data cannot be larger than capacity");
 372 
 373     verify(r, r->garbage() <= r->capacity(),
 374            "Garbage cannot be larger than capacity");
 375 
 376     verify(r, r->used() <= r->capacity(),
 377            "Used cannot be larger than capacity");
 378 
 379     verify(r, r->get_shared_allocs() <= r->capacity(),
 380            "Shared alloc count should not be larger than capacity");
 381 
 382     verify(r, r->get_tlab_allocs() <= r->capacity(),
 383            "TLAB alloc count should not be larger than capacity");
 384 
 385     verify(r, r->get_gclab_allocs() <= r->capacity(),
 386            "GCLAB alloc count should not be larger than capacity");
 387 
 388     verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(),
 389            "Accurate accounting: shared + TLAB + GCLAB = used");
 390 
 391     verify(r, !r->is_empty() || !r->has_live(),
 392            "Empty regions should not have live data");
 393 
 394     verify(r, r->is_cset() == _heap->collection_set()->is_in(r),
 395            "Transitional: region flags and collection set agree");
 396 
 397     verify(r, r->is_empty() || r->seqnum_first_alloc() != 0,
 398            "Non-empty regions should have first seqnum set");
 399 
 400     verify(r, r->is_empty() || (r->seqnum_first_alloc_mutator() != 0 || r->seqnum_first_alloc_gc() != 0),
 401            "Non-empty regions should have first seqnum set to either GC or mutator");
 402 
 403     verify(r, r->is_empty() || r->seqnum_last_alloc() != 0,
 404            "Non-empty regions should have last seqnum set");
 405 
 406     verify(r, r->is_empty() || (r->seqnum_last_alloc_mutator() != 0 || r->seqnum_last_alloc_gc() != 0),
 407            "Non-empty regions should have last seqnum set to either GC or mutator");
 408 
 409     verify(r, r->seqnum_first_alloc() <= r->seqnum_last_alloc(),
 410            "First seqnum should not be greater than last timestamp");
 411 
 412     verify(r, r->seqnum_first_alloc_mutator() <= r->seqnum_last_alloc_mutator(),
 413            "First mutator seqnum should not be greater than last seqnum");
 414 
 415     verify(r, r->seqnum_first_alloc_gc() <= r->seqnum_last_alloc_gc(),
 416            "First GC seqnum should not be greater than last seqnum");
 417   }
 418 };
 419 
 420 class ShenandoahVerifierReachableTask : public AbstractGangTask {
 421 private:
 422   const char* _label;
 423   ShenandoahRootVerifier* _verifier;
 424   ShenandoahVerifier::VerifyOptions _options;
 425   ShenandoahHeap* _heap;
 426   ShenandoahLivenessData* _ld;
 427   MarkBitMap* _bitmap;
 428   volatile size_t _processed;
 429 
 430 public:
 431   ShenandoahVerifierReachableTask(MarkBitMap* bitmap,
 432                                   ShenandoahLivenessData* ld,
 433                                   ShenandoahRootVerifier* verifier,
 434                                   const char* label,
 435                                   ShenandoahVerifier::VerifyOptions options) :
 436     AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"),
 437     _label(label),
 438     _verifier(verifier),
 439     _options(options),
 440     _heap(ShenandoahHeap::heap()),
 441     _ld(ld),
 442     _bitmap(bitmap),
 443     _processed(0) {};
 444 
 445   size_t processed() {
 446     return _processed;
 447   }
 448 
 449   virtual void work(uint worker_id) {
 450     ResourceMark rm;
 451     ShenandoahVerifierStack stack;
 452 
 453     // On level 2, we need to only check the roots once.
 454     // On level 3, we want to check the roots, and seed the local stack.
 455     // It is a lesser evil to accept multiple root scans at level 3, because
 456     // extended parallelism would buy us out.
 457     if (((ShenandoahVerifyLevel == 2) && (worker_id == 0))
 458         || (ShenandoahVerifyLevel >= 3)) {
 459         ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
 460                                       ShenandoahMessageBuffer("%s, Roots", _label),
 461                                       _options);
 462         if (_heap->unload_classes()) {
 463           _verifier->strong_roots_do(&cl);
 464         } else {
 465           _verifier->roots_do(&cl);
 466         }
 467     }
 468 
 469     size_t processed = 0;
 470 
 471     if (ShenandoahVerifyLevel >= 3) {
 472       ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
 473                                     ShenandoahMessageBuffer("%s, Reachable", _label),
 474                                     _options);
 475       while (!stack.is_empty()) {
 476         processed++;
 477         ShenandoahVerifierTask task = stack.pop();
 478         cl.verify_oops_from(task.obj());
 479       }
 480     }
 481 
 482     Atomic::add(processed, &_processed);
 483   }
 484 };
 485 
 486 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask {
 487 private:
 488   const char* _label;
 489   ShenandoahVerifier::VerifyOptions _options;
 490   ShenandoahHeap *_heap;
 491   MarkBitMap* _bitmap;
 492   ShenandoahLivenessData* _ld;
 493   volatile size_t _claimed;
 494   volatile size_t _processed;
 495 
 496 public:
 497   ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap,
 498                                      ShenandoahLivenessData* ld,
 499                                      const char* label,
 500                                      ShenandoahVerifier::VerifyOptions options) :
 501           AbstractGangTask("Shenandoah Parallel Verifier Marked Region"),
 502           _label(label),
 503           _options(options),
 504           _heap(ShenandoahHeap::heap()),
 505           _bitmap(bitmap),
 506           _ld(ld),
 507           _claimed(0),
 508           _processed(0) {};
 509 
 510   size_t processed() {
 511     return _processed;
 512   }
 513 
 514   virtual void work(uint worker_id) {
 515     ShenandoahVerifierStack stack;
 516     ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld,
 517                                   ShenandoahMessageBuffer("%s, Marked", _label),
 518                                   _options);
 519 
 520     while (true) {
 521       size_t v = Atomic::add(1u, &_claimed) - 1;
 522       if (v < _heap->num_regions()) {
 523         ShenandoahHeapRegion* r = _heap->get_region(v);
 524         if (!r->is_humongous() && !r->is_trash()) {
 525           work_regular(r, stack, cl);
 526         } else if (r->is_humongous_start()) {
 527           work_humongous(r, stack, cl);
 528         }
 529       } else {
 530         break;
 531       }
 532     }
 533   }
 534 
 535   virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
 536     size_t processed = 0;
 537     HeapWord* obj = r->bottom();
 538     if (_heap->complete_marking_context()->is_marked((oop)obj)) {
 539       verify_and_follow(obj, stack, cl, &processed);
 540     }
 541     Atomic::add(processed, &_processed);
 542   }
 543 
 544   virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
 545     size_t processed = 0;
 546     MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
 547     HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
 548 
 549     // Bitmaps, before TAMS
 550     if (tams > r->bottom()) {
 551       HeapWord* start = r->bottom();
 552       HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
 553 
 554       while (addr < tams) {
 555         verify_and_follow(addr, stack, cl, &processed);
 556         addr += 1;
 557         if (addr < tams) {
 558           addr = mark_bit_map->get_next_marked_addr(addr, tams);
 559         }
 560       }
 561     }
 562 
 563     // Size-based, after TAMS
 564     {
 565       HeapWord* limit = r->top();
 566       HeapWord* addr = tams;
 567 
 568       while (addr < limit) {
 569         verify_and_follow(addr, stack, cl, &processed);
 570         addr += oop(addr)->size();
 571       }
 572     }
 573 
 574     Atomic::add(processed, &_processed);
 575   }
 576 
 577   void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
 578     if (!_bitmap->par_mark(addr)) return;
 579 
 580     // Verify the object itself:
 581     oop obj = oop(addr);
 582     cl.verify_oop_standalone(obj);
 583 
 584     // Verify everything reachable from that object too, hopefully realizing
 585     // everything was already marked, and never touching further:
 586     cl.verify_oops_from(obj);
 587     (*processed)++;
 588 
 589     while (!stack.is_empty()) {
 590       ShenandoahVerifierTask task = stack.pop();
 591       cl.verify_oops_from(task.obj());
 592       (*processed)++;
 593     }
 594   }
 595 };
 596 
 597 class VerifyThreadGCState : public ThreadClosure {
 598 private:
 599   const char* _label;
 600   char _expected;
 601 
 602 public:
 603   VerifyThreadGCState(const char* label, char expected) : _expected(expected) {}
 604   void do_thread(Thread* t) {
 605     char actual = ShenandoahThreadLocalData::gc_state(t);
 606     if (actual != _expected) {
 607       fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual);
 608     }
 609   }
 610 };
 611 
 612 class ShenandoahGCStateResetter : public StackObj {
 613 private:
 614   ShenandoahHeap* const _heap;
 615   char _gc_state;
 616 
 617 public:
 618   ShenandoahGCStateResetter() : _heap(ShenandoahHeap::heap()) {
 619     _gc_state = _heap->gc_state();
 620     _heap->_gc_state.clear();
 621   }
 622 
 623   ~ShenandoahGCStateResetter() {
 624     _heap->_gc_state.set(_gc_state);
 625     assert(_heap->gc_state() == _gc_state, "Should be restored");
 626   }
 627 };
 628 
 629 void ShenandoahVerifier::verify_at_safepoint(const char *label,
 630                                              VerifyForwarded forwarded, VerifyMarked marked,
 631                                              VerifyCollectionSet cset,
 632                                              VerifyLiveness liveness, VerifyRegions regions,
 633                                              VerifyGCState gcstate) {
 634   guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
 635   guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
 636 
 637   // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
 638   ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
 639 
 640   log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel);
 641 
 642   // GC state checks
 643   {
 644     char expected = -1;
 645     bool enabled;
 646     switch (gcstate) {
 647       case _verify_gcstate_disable:
 648         enabled = false;
 649         break;
 650       case _verify_gcstate_forwarded:
 651         enabled = true;
 652         expected = ShenandoahHeap::HAS_FORWARDED;
 653         break;
 654       case _verify_gcstate_evacuation:
 655         enabled = true;
 656         expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION;
 657         break;
 658       case _verify_gcstate_stable:
 659         enabled = true;
 660         expected = ShenandoahHeap::STABLE;
 661         break;
 662       default:
 663         enabled = false;
 664         assert(false, "Unhandled gc-state verification");
 665     }
 666 
 667     if (enabled) {
 668       char actual = _heap->gc_state();
 669       if (actual != expected) {
 670         fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual);
 671       }
 672 
 673       VerifyThreadGCState vtgcs(label, expected);
 674       Threads::java_threads_do(&vtgcs);
 675     }
 676   }
 677 
 678   // Deactivate barriers temporarily: Verifier wants plain heap accesses
 679   ShenandoahGCStateResetter resetter;
 680 
 681   // Heap size checks
 682   {
 683     ShenandoahHeapLocker lock(_heap->lock());
 684 
 685     ShenandoahCalculateRegionStatsClosure cl;
 686     _heap->heap_region_iterate(&cl);
 687     size_t heap_used = _heap->used();
 688     guarantee(cl.used() == heap_used,
 689               "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s",
 690               label,
 691               byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used),
 692               byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used()));
 693 
 694     size_t heap_committed = _heap->committed();
 695     guarantee(cl.committed() == heap_committed,
 696               "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "%s, regions-committed = " SIZE_FORMAT "%s",
 697               label,
 698               byte_size_in_proper_unit(heap_committed), proper_unit_for_byte_size(heap_committed),
 699               byte_size_in_proper_unit(cl.committed()), proper_unit_for_byte_size(cl.committed()));
 700   }
 701 
 702   // Internal heap region checks
 703   if (ShenandoahVerifyLevel >= 1) {
 704     ShenandoahVerifyHeapRegionClosure cl(label, regions);
 705     _heap->heap_region_iterate(&cl);
 706   }
 707 
 708   OrderAccess::fence();
 709   _heap->make_parsable(false);
 710 
 711   // Allocate temporary bitmap for storing marking wavefront:
 712   _verification_bit_map->clear();
 713 
 714   // Allocate temporary array for storing liveness data
 715   ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC);
 716   Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0);
 717 
 718   const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, cset, liveness, regions, gcstate);
 719 
 720   // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap.
 721   // This verifies what application can see, since it only cares about reachable objects.
 722   size_t count_reachable = 0;
 723   if (ShenandoahVerifyLevel >= 2) {
 724     ShenandoahRootVerifier verifier;
 725 
 726     ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &verifier, label, options);
 727     _heap->workers()->run_task(&task);
 728     count_reachable = task.processed();
 729   }
 730 
 731   // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector,
 732   // not the application, can see during the region scans. There is no reason to process the objects
 733   // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS:
 734   // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics
 735   // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect)
 736   // version
 737 
 738   size_t count_marked = 0;
 739   if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
 740     guarantee(_heap->marking_context()->is_complete(), "Marking context should be complete");
 741     ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options);
 742     _heap->workers()->run_task(&task);
 743     count_marked = task.processed();
 744   } else {
 745     guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_incomplete || marked == _verify_marked_disable, "Should be");
 746   }
 747 
 748   // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes
 749   // marked objects.
 750 
 751   if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) {
 752     for (size_t i = 0; i < _heap->num_regions(); i++) {
 753       ShenandoahHeapRegion* r = _heap->get_region(i);
 754 
 755       juint verf_live = 0;
 756       if (r->is_humongous()) {
 757         // For humongous objects, test if start region is marked live, and if so,
 758         // all humongous regions in that chain have live data equal to their "used".
 759         juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]);
 760         if (start_live > 0) {
 761           verf_live = (juint)(r->used() / HeapWordSize);
 762         }
 763       } else {
 764         verf_live = OrderAccess::load_acquire(&ld[r->region_number()]);
 765       }
 766 
 767       size_t reg_live = r->get_live_data_words();
 768       if (reg_live != verf_live) {
 769         ResourceMark rm;
 770         stringStream ss;
 771         r->print_on(&ss);
 772         fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s",
 773               label, reg_live, verf_live, ss.as_string());
 774       }
 775     }
 776   }
 777 
 778   log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)",
 779                label, ShenandoahVerifyLevel, count_reachable, count_marked);
 780 
 781   FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld);
 782 }
 783 
 784 void ShenandoahVerifier::verify_generic(VerifyOption vo) {
 785   verify_at_safepoint(
 786           "Generic Verification",
 787           _verify_forwarded_allow,     // conservatively allow forwarded
 788           _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
 789           _verify_cset_disable,        // cset may be inconsistent
 790           _verify_liveness_disable,    // no reliable liveness data
 791           _verify_regions_disable,     // no reliable region data
 792           _verify_gcstate_disable      // no data about gcstate
 793   );
 794 }
 795 
 796 void ShenandoahVerifier::verify_before_concmark() {
 797   if (_heap->has_forwarded_objects()) {
 798     verify_at_safepoint(
 799             "Before Mark",
 800             _verify_forwarded_allow,     // may have forwarded references
 801             _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
 802             _verify_cset_forwarded,      // allow forwarded references to cset
 803             _verify_liveness_disable,    // no reliable liveness data
 804             _verify_regions_notrash,     // no trash regions
 805             _verify_gcstate_forwarded    // there are forwarded objects
 806     );
 807   } else {
 808     verify_at_safepoint(
 809             "Before Mark",
 810             _verify_forwarded_none,      // UR should have fixed up
 811             _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
 812             _verify_cset_none,           // UR should have fixed this
 813             _verify_liveness_disable,    // no reliable liveness data
 814             _verify_regions_notrash,     // no trash regions
 815             _verify_gcstate_stable       // there are no forwarded objects
 816     );
 817   }
 818 }
 819 
 820 void ShenandoahVerifier::verify_after_concmark() {
 821   verify_at_safepoint(
 822           "After Mark",
 823           _verify_forwarded_none,      // no forwarded references
 824           _verify_marked_complete,     // bitmaps as precise as we can get
 825           _verify_cset_none,           // no references to cset anymore
 826           _verify_liveness_complete,   // liveness data must be complete here
 827           _verify_regions_disable,     // trash regions not yet recycled
 828           _verify_gcstate_stable       // mark should have stabilized the heap
 829   );
 830 }
 831 
 832 void ShenandoahVerifier::verify_before_evacuation() {
 833   verify_at_safepoint(
 834           "Before Evacuation",
 835           _verify_forwarded_none,    // no forwarded references
 836           _verify_marked_complete,   // walk over marked objects too
 837           _verify_cset_disable,      // non-forwarded references to cset expected
 838           _verify_liveness_complete, // liveness data must be complete here
 839           _verify_regions_disable,   // trash regions not yet recycled
 840           _verify_gcstate_stable     // mark should have stabilized the heap
 841   );
 842 }
 843 
 844 void ShenandoahVerifier::verify_during_evacuation() {
 845   verify_at_safepoint(
 846           "During Evacuation",
 847           _verify_forwarded_allow,   // some forwarded references are allowed
 848           _verify_marked_disable,    // walk only roots
 849           _verify_cset_disable,      // some cset references are not forwarded yet
 850           _verify_liveness_disable,  // liveness data might be already stale after pre-evacs
 851           _verify_regions_disable,   // trash regions not yet recycled
 852           _verify_gcstate_evacuation // evacuation is in progress
 853   );
 854 }
 855 
 856 void ShenandoahVerifier::verify_after_evacuation() {
 857   verify_at_safepoint(
 858           "After Evacuation",
 859           _verify_forwarded_allow,     // objects are still forwarded
 860           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
 861           _verify_cset_forwarded,      // all cset refs are fully forwarded
 862           _verify_liveness_disable,    // no reliable liveness data anymore
 863           _verify_regions_notrash,     // trash regions have been recycled already
 864           _verify_gcstate_forwarded    // evacuation produced some forwarded objects
 865   );
 866 }
 867 
 868 void ShenandoahVerifier::verify_before_updaterefs() {
 869   verify_at_safepoint(
 870           "Before Updating References",
 871           _verify_forwarded_allow,     // forwarded references allowed
 872           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
 873           _verify_cset_forwarded,      // all cset refs are fully forwarded
 874           _verify_liveness_disable,    // no reliable liveness data anymore
 875           _verify_regions_notrash,     // trash regions have been recycled already
 876           _verify_gcstate_forwarded    // evacuation should have produced some forwarded objects
 877   );
 878 }
 879 
 880 void ShenandoahVerifier::verify_after_updaterefs() {
 881   verify_at_safepoint(
 882           "After Updating References",
 883           _verify_forwarded_none,      // no forwarded references
 884           _verify_marked_complete,     // bitmaps might be stale, but alloc-after-mark should be well
 885           _verify_cset_none,           // no cset references, all updated
 886           _verify_liveness_disable,    // no reliable liveness data anymore
 887           _verify_regions_nocset,      // no cset regions, trash regions have appeared
 888           _verify_gcstate_stable       // update refs had cleaned up forwarded objects
 889   );
 890 }
 891 
 892 void ShenandoahVerifier::verify_after_degenerated() {
 893   verify_at_safepoint(
 894           "After Degenerated GC",
 895           _verify_forwarded_none,      // all objects are non-forwarded
 896           _verify_marked_complete,     // all objects are marked in complete bitmap
 897           _verify_cset_none,           // no cset references
 898           _verify_liveness_disable,    // no reliable liveness data anymore
 899           _verify_regions_notrash_nocset, // no trash, no cset
 900           _verify_gcstate_stable       // degenerated refs had cleaned up forwarded objects
 901   );
 902 }
 903 
 904 void ShenandoahVerifier::verify_before_traversal() {
 905   verify_at_safepoint(
 906           "Before Traversal",
 907           _verify_forwarded_none,      // cannot have forwarded objects
 908           _verify_marked_disable,      // bitmaps are not relevant before traversal
 909           _verify_cset_none,           // no cset references before traversal
 910           _verify_liveness_disable,    // no reliable liveness data anymore
 911           _verify_regions_notrash_nocset, // no trash and no cset regions
 912           _verify_gcstate_stable       // nothing forwarded before traversal
 913   );
 914 }
 915 
 916 void ShenandoahVerifier::verify_after_traversal() {
 917   verify_at_safepoint(
 918           "After Traversal",
 919           _verify_forwarded_none,      // cannot have forwarded objects
 920           _verify_marked_complete,     // should have complete marking after traversal
 921           _verify_cset_none,           // no cset references left after traversal
 922           _verify_liveness_disable,    // liveness data is not collected for new allocations
 923           _verify_regions_nocset,      // no cset regions, trash regions allowed
 924           _verify_gcstate_stable       // nothing forwarded after traversal
 925   );
 926 }
 927 
 928 void ShenandoahVerifier::verify_before_fullgc() {
 929   verify_at_safepoint(
 930           "Before Full GC",
 931           _verify_forwarded_allow,     // can have forwarded objects
 932           _verify_marked_disable,      // do not verify marked: lots ot time wasted checking dead allocations
 933           _verify_cset_disable,        // cset might be foobared
 934           _verify_liveness_disable,    // no reliable liveness data anymore
 935           _verify_regions_disable,     // no reliable region data here
 936           _verify_gcstate_disable      // no reliable gcstate data
 937   );
 938 }
 939 
 940 void ShenandoahVerifier::verify_after_fullgc() {
 941   verify_at_safepoint(
 942           "After Full GC",
 943           _verify_forwarded_none,      // all objects are non-forwarded
 944           _verify_marked_complete,     // all objects are marked in complete bitmap
 945           _verify_cset_none,           // no cset references
 946           _verify_liveness_disable,    // no reliable liveness data anymore
 947           _verify_regions_notrash_nocset, // no trash, no cset
 948           _verify_gcstate_stable       // full gc cleaned up everything
 949   );
 950 }
 951 
 952 class ShenandoahVerifyNoForwared : public OopClosure {
 953 private:
 954   template <class T>
 955   void do_oop_work(T* p) {
 956     T o = RawAccess<>::oop_load(p);
 957     if (!CompressedOops::is_null(o)) {
 958       oop obj = CompressedOops::decode_not_null(o);
 959       oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
 960       if (obj != fwd) {
 961         ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
 962                                          "Verify Roots", "Should not be forwarded", __FILE__, __LINE__);
 963       }
 964     }
 965   }
 966 
 967 public:
 968   void do_oop(narrowOop* p) { do_oop_work(p); }
 969   void do_oop(oop* p)       { do_oop_work(p); }
 970 };
 971 
 972 class ShenandoahVerifyInToSpaceClosure : public OopClosure {
 973 private:
 974   template <class T>
 975   void do_oop_work(T* p) {
 976     T o = RawAccess<>::oop_load(p);
 977     if (!CompressedOops::is_null(o)) {
 978       oop obj = CompressedOops::decode_not_null(o);
 979       ShenandoahHeap* heap = ShenandoahHeap::heap_no_check();
 980 
 981       if (!heap->marking_context()->is_marked(obj)) {
 982         ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
 983                 "Verify Roots In To-Space", "Should be marked", __FILE__, __LINE__);
 984       }
 985 
 986       if (heap->in_collection_set(obj)) {
 987         ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
 988                 "Verify Roots In To-Space", "Should not be in collection set", __FILE__, __LINE__);
 989       }
 990 
 991       oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
 992       if (obj != fwd) {
 993         ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
 994                 "Verify Roots In To-Space", "Should not be forwarded", __FILE__, __LINE__);
 995       }
 996     }
 997   }
 998 
 999 public:
1000   void do_oop(narrowOop* p) { do_oop_work(p); }
1001   void do_oop(oop* p)       { do_oop_work(p); }
1002 };
1003 
1004 void ShenandoahVerifier::verify_roots_in_to_space() {
1005   ShenandoahRootVerifier verifier;
1006   ShenandoahVerifyInToSpaceClosure cl;
1007   verifier.oops_do(&cl);
1008 }
1009 
1010 void ShenandoahVerifier::verify_roots_no_forwarded() {
1011   ShenandoahRootVerifier verifier;
1012   ShenandoahVerifyNoForwared cl;
1013   verifier.oops_do(&cl);
1014 }
1015 
1016 void ShenandoahVerifier::verify_roots_no_forwarded_except(ShenandoahRootVerifier::RootTypes types) {
1017   ShenandoahRootVerifier verifier;
1018   verifier.excludes(types);
1019   ShenandoahVerifyNoForwared cl;
1020   verifier.oops_do(&cl);
1021 }