1 /*
   2  * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/g1/g1Allocator.inline.hpp"
  27 #include "gc/g1/g1CollectedHeap.inline.hpp"
  28 #include "gc/g1/g1ConcurrentMarkThread.hpp"
  29 #include "gc/g1/g1HeapVerifier.hpp"
  30 #include "gc/g1/g1Policy.hpp"
  31 #include "gc/g1/g1RemSet.hpp"
  32 #include "gc/g1/g1RootProcessor.hpp"
  33 #include "gc/g1/heapRegion.inline.hpp"
  34 #include "gc/g1/heapRegionRemSet.hpp"
  35 #include "gc/g1/g1StringDedup.hpp"
  36 #include "logging/log.hpp"
  37 #include "logging/logStream.hpp"
  38 #include "memory/iterator.inline.hpp"
  39 #include "memory/resourceArea.hpp"
  40 #include "oops/access.inline.hpp"
  41 #include "oops/compressedOops.inline.hpp"
  42 #include "oops/oop.inline.hpp"
  43 #include "runtime/handles.inline.hpp"
  44 
  45 int G1HeapVerifier::_enabled_verification_types = G1HeapVerifier::G1VerifyAll;
  46 
  47 class VerifyRootsClosure: public OopClosure {
  48 private:
  49   G1CollectedHeap* _g1h;
  50   VerifyOption     _vo;
  51   bool             _failures;
  52 public:
  53   // _vo == UsePrevMarking -> use "prev" marking information,
  54   // _vo == UseNextMarking -> use "next" marking information,
  55   // _vo == UseFullMarking -> use "next" marking bitmap but no TAMS
  56   VerifyRootsClosure(VerifyOption vo) :
  57     _g1h(G1CollectedHeap::heap()),
  58     _vo(vo),
  59     _failures(false) { }
  60 
  61   bool failures() { return _failures; }
  62 
  63   template <class T> void do_oop_work(T* p) {
  64     T heap_oop = RawAccess<>::oop_load(p);
  65     if (!CompressedOops::is_null(heap_oop)) {
  66       oop obj = CompressedOops::decode_not_null(heap_oop);
  67       if (_g1h->is_obj_dead_cond(obj, _vo)) {
  68         Log(gc, verify) log;
  69         log.error("Root location " PTR_FORMAT " points to dead obj " PTR_FORMAT, p2i(p), p2i(obj));
  70         ResourceMark rm;
  71         LogStream ls(log.error());
  72         obj->print_on(&ls);
  73         _failures = true;
  74       }
  75     }
  76   }
  77 
  78   void do_oop(oop* p)       { do_oop_work(p); }
  79   void do_oop(narrowOop* p) { do_oop_work(p); }
  80 };
  81 
  82 class G1VerifyCodeRootOopClosure: public OopClosure {
  83   G1CollectedHeap* _g1h;
  84   OopClosure* _root_cl;
  85   nmethod* _nm;
  86   VerifyOption _vo;
  87   bool _failures;
  88 
  89   template <class T> void do_oop_work(T* p) {
  90     // First verify that this root is live
  91     _root_cl->do_oop(p);
  92 
  93     if (!G1VerifyHeapRegionCodeRoots) {
  94       // We're not verifying the code roots attached to heap region.
  95       return;
  96     }
  97 
  98     // Don't check the code roots during marking verification in a full GC
  99     if (_vo == VerifyOption_G1UseFullMarking) {
 100       return;
 101     }
 102 
 103     // Now verify that the current nmethod (which contains p) is
 104     // in the code root list of the heap region containing the
 105     // object referenced by p.
 106 
 107     T heap_oop = RawAccess<>::oop_load(p);
 108     if (!CompressedOops::is_null(heap_oop)) {
 109       oop obj = CompressedOops::decode_not_null(heap_oop);
 110 
 111       // Now fetch the region containing the object
 112       HeapRegion* hr = _g1h->heap_region_containing(obj);
 113       HeapRegionRemSet* hrrs = hr->rem_set();
 114       // Verify that the strong code root list for this region
 115       // contains the nmethod
 116       if (!hrrs->strong_code_roots_list_contains(_nm)) {
 117         log_error(gc, verify)("Code root location " PTR_FORMAT " "
 118                               "from nmethod " PTR_FORMAT " not in strong "
 119                               "code roots for region [" PTR_FORMAT "," PTR_FORMAT ")",
 120                               p2i(p), p2i(_nm), p2i(hr->bottom()), p2i(hr->end()));
 121         _failures = true;
 122       }
 123     }
 124   }
 125 
 126 public:
 127   G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo):
 128     _g1h(g1h), _root_cl(root_cl), _nm(NULL), _vo(vo), _failures(false) {}
 129 
 130   void do_oop(oop* p) { do_oop_work(p); }
 131   void do_oop(narrowOop* p) { do_oop_work(p); }
 132 
 133   void set_nmethod(nmethod* nm) { _nm = nm; }
 134   bool failures() { return _failures; }
 135 };
 136 
 137 class G1VerifyCodeRootBlobClosure: public CodeBlobClosure {
 138   G1VerifyCodeRootOopClosure* _oop_cl;
 139 
 140 public:
 141   G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl):
 142     _oop_cl(oop_cl) {}
 143 
 144   void do_code_blob(CodeBlob* cb) {
 145     nmethod* nm = cb->as_nmethod_or_null();
 146     if (nm != NULL) {
 147       _oop_cl->set_nmethod(nm);
 148       nm->oops_do(_oop_cl);
 149     }
 150   }
 151 };
 152 
 153 class YoungRefCounterClosure : public OopClosure {
 154   G1CollectedHeap* _g1h;
 155   int              _count;
 156  public:
 157   YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {}
 158   void do_oop(oop* p)       { if (_g1h->is_in_young(*p)) { _count++; } }
 159   void do_oop(narrowOop* p) { ShouldNotReachHere(); }
 160 
 161   int count() { return _count; }
 162   void reset_count() { _count = 0; };
 163 };
 164 
 165 class VerifyCLDClosure: public CLDClosure {
 166   YoungRefCounterClosure _young_ref_counter_closure;
 167   OopClosure *_oop_closure;
 168  public:
 169   VerifyCLDClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {}
 170   void do_cld(ClassLoaderData* cld) {
 171     cld->oops_do(_oop_closure, ClassLoaderData::_claim_none);
 172 
 173     _young_ref_counter_closure.reset_count();
 174     cld->oops_do(&_young_ref_counter_closure, ClassLoaderData::_claim_none);
 175     if (_young_ref_counter_closure.count() > 0) {
 176       guarantee(cld->has_modified_oops(), "CLD " PTR_FORMAT ", has young %d refs but is not dirty.", p2i(cld), _young_ref_counter_closure.count());
 177     }
 178   }
 179 };
 180 
 181 class VerifyLivenessOopClosure: public BasicOopIterateClosure {
 182   G1CollectedHeap* _g1h;
 183   VerifyOption _vo;
 184 public:
 185   VerifyLivenessOopClosure(G1CollectedHeap* g1h, VerifyOption vo):
 186     _g1h(g1h), _vo(vo)
 187   { }
 188   void do_oop(narrowOop *p) { do_oop_work(p); }
 189   void do_oop(      oop *p) { do_oop_work(p); }
 190 
 191   template <class T> void do_oop_work(T *p) {
 192     oop obj = RawAccess<>::oop_load(p);
 193     guarantee(obj == NULL || !_g1h->is_obj_dead_cond(obj, _vo),
 194               "Dead object referenced by a not dead object");
 195   }
 196 };
 197 
 198 class VerifyObjsInRegionClosure: public ObjectClosure {
 199 private:
 200   G1CollectedHeap* _g1h;
 201   size_t _live_bytes;
 202   HeapRegion *_hr;
 203   VerifyOption _vo;
 204 public:
 205   // _vo == UsePrevMarking -> use "prev" marking information,
 206   // _vo == UseNextMarking -> use "next" marking information,
 207   // _vo == UseFullMarking -> use "next" marking bitmap but no TAMS.
 208   VerifyObjsInRegionClosure(HeapRegion *hr, VerifyOption vo)
 209     : _live_bytes(0), _hr(hr), _vo(vo) {
 210     _g1h = G1CollectedHeap::heap();
 211   }
 212   void do_object(oop o) {
 213     VerifyLivenessOopClosure isLive(_g1h, _vo);
 214     assert(o != NULL, "Huh?");
 215     if (!_g1h->is_obj_dead_cond(o, _vo)) {
 216       // If the object is alive according to the full gc mark,
 217       // then verify that the marking information agrees.
 218       // Note we can't verify the contra-positive of the
 219       // above: if the object is dead (according to the mark
 220       // word), it may not be marked, or may have been marked
 221       // but has since became dead, or may have been allocated
 222       // since the last marking.
 223       if (_vo == VerifyOption_G1UseFullMarking) {
 224         guarantee(!_g1h->is_obj_dead(o), "Full GC marking and concurrent mark mismatch");
 225       }
 226 
 227       o->oop_iterate(&isLive);
 228       if (!_hr->obj_allocated_since_prev_marking(o)) {
 229         size_t obj_size = o->size();    // Make sure we don't overflow
 230         _live_bytes += (obj_size * HeapWordSize);
 231       }
 232     }
 233   }
 234   size_t live_bytes() { return _live_bytes; }
 235 };
 236 
 237 class VerifyArchiveOopClosure: public BasicOopIterateClosure {
 238   HeapRegion* _hr;
 239 public:
 240   VerifyArchiveOopClosure(HeapRegion *hr)
 241     : _hr(hr) { }
 242   void do_oop(narrowOop *p) { do_oop_work(p); }
 243   void do_oop(      oop *p) { do_oop_work(p); }
 244 
 245   template <class T> void do_oop_work(T *p) {
 246     oop obj = RawAccess<>::oop_load(p);
 247 
 248     if (_hr->is_open_archive()) {
 249       guarantee(obj == NULL || G1ArchiveAllocator::is_archived_object(obj),
 250                 "Archive object at " PTR_FORMAT " references a non-archive object at " PTR_FORMAT,
 251                 p2i(p), p2i(obj));
 252     } else {
 253       assert(_hr->is_closed_archive(), "should be closed archive region");
 254       guarantee(obj == NULL || G1ArchiveAllocator::is_closed_archive_object(obj),
 255                 "Archive object at " PTR_FORMAT " references a non-archive object at " PTR_FORMAT,
 256                 p2i(p), p2i(obj));
 257     }
 258   }
 259 };
 260 
 261 class VerifyObjectInArchiveRegionClosure: public ObjectClosure {
 262   HeapRegion* _hr;
 263 public:
 264   VerifyObjectInArchiveRegionClosure(HeapRegion *hr, bool verbose)
 265     : _hr(hr) { }
 266   // Verify that all object pointers are to archive regions.
 267   void do_object(oop o) {
 268     VerifyArchiveOopClosure checkOop(_hr);
 269     assert(o != NULL, "Should not be here for NULL oops");
 270     o->oop_iterate(&checkOop);
 271   }
 272 };
 273 
 274 // Should be only used at CDS dump time
 275 class VerifyArchivePointerRegionClosure: public HeapRegionClosure {
 276 private:
 277   G1CollectedHeap* _g1h;
 278 public:
 279   VerifyArchivePointerRegionClosure(G1CollectedHeap* g1h) { }
 280   virtual bool do_heap_region(HeapRegion* r) {
 281    if (r->is_archive()) {
 282       VerifyObjectInArchiveRegionClosure verify_oop_pointers(r, false);
 283       r->object_iterate(&verify_oop_pointers);
 284     }
 285     return false;
 286   }
 287 };
 288 
 289 void G1HeapVerifier::verify_archive_regions() {
 290   G1CollectedHeap*  g1h = G1CollectedHeap::heap();
 291   VerifyArchivePointerRegionClosure cl(NULL);
 292   g1h->heap_region_iterate(&cl);
 293 }
 294 
 295 class VerifyRegionClosure: public HeapRegionClosure {
 296 private:
 297   bool             _par;
 298   VerifyOption     _vo;
 299   bool             _failures;
 300 public:
 301   // _vo == UsePrevMarking -> use "prev" marking information,
 302   // _vo == UseNextMarking -> use "next" marking information,
 303   // _vo == UseFullMarking -> use "next" marking bitmap but no TAMS
 304   VerifyRegionClosure(bool par, VerifyOption vo)
 305     : _par(par),
 306       _vo(vo),
 307       _failures(false) {}
 308 
 309   bool failures() {
 310     return _failures;
 311   }
 312 
 313   bool do_heap_region(HeapRegion* r) {
 314     guarantee(!r->is_young() || r->rem_set()->is_complete(), "Remembered set for Young region %u must be complete, is %s", r->hrm_index(), r->rem_set()->get_state_str());
 315     // Humongous and old regions regions might be of any state, so can't check here.
 316     guarantee(!r->is_free() || !r->rem_set()->is_tracked(), "Remembered set for free region %u must be untracked, is %s", r->hrm_index(), r->rem_set()->get_state_str());
 317     // Verify that the continues humongous regions' remembered set state matches the
 318     // one from the starts humongous region.
 319     if (r->is_continues_humongous()) {
 320       if (r->rem_set()->get_state_str() != r->humongous_start_region()->rem_set()->get_state_str()) {
 321          log_error(gc, verify)("Remset states differ: Region %u (%s) remset %s with starts region %u (%s) remset %s",
 322                                r->hrm_index(),
 323                                r->get_short_type_str(),
 324                                r->rem_set()->get_state_str(),
 325                                r->humongous_start_region()->hrm_index(),
 326                                r->humongous_start_region()->get_short_type_str(),
 327                                r->humongous_start_region()->rem_set()->get_state_str());
 328          _failures = true;
 329       }
 330     }
 331     // For archive regions, verify there are no heap pointers to
 332     // non-pinned regions. For all others, verify liveness info.
 333     if (r->is_closed_archive()) {
 334       VerifyObjectInArchiveRegionClosure verify_oop_pointers(r, false);
 335       r->object_iterate(&verify_oop_pointers);
 336       return true;
 337     } else if (r->is_open_archive()) {
 338       VerifyObjsInRegionClosure verify_open_archive_oop(r, _vo);
 339       r->object_iterate(&verify_open_archive_oop);
 340       return true;
 341     } else if (!r->is_continues_humongous()) {
 342       bool failures = false;
 343       r->verify(_vo, &failures);
 344       if (failures) {
 345         _failures = true;
 346       } else if (!r->is_starts_humongous()) {
 347         VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
 348         r->object_iterate(&not_dead_yet_cl);
 349         if (_vo != VerifyOption_G1UseNextMarking) {
 350           if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
 351             log_error(gc, verify)("[" PTR_FORMAT "," PTR_FORMAT "] max_live_bytes " SIZE_FORMAT " < calculated " SIZE_FORMAT,
 352                                   p2i(r->bottom()), p2i(r->end()), r->max_live_bytes(), not_dead_yet_cl.live_bytes());
 353             _failures = true;
 354           }
 355         } else {
 356           // When vo == UseNextMarking we cannot currently do a sanity
 357           // check on the live bytes as the calculation has not been
 358           // finalized yet.
 359         }
 360       }
 361     }
 362     return false; // stop the region iteration if we hit a failure
 363   }
 364 };
 365 
 366 // This is the task used for parallel verification of the heap regions
 367 
 368 class G1ParVerifyTask: public AbstractGangTask {
 369 private:
 370   G1CollectedHeap*  _g1h;
 371   VerifyOption      _vo;
 372   bool              _failures;
 373   HeapRegionClaimer _hrclaimer;
 374 
 375 public:
 376   // _vo == UsePrevMarking -> use "prev" marking information,
 377   // _vo == UseNextMarking -> use "next" marking information,
 378   // _vo == UseFullMarking -> use "next" marking bitmap but no TAMS
 379   G1ParVerifyTask(G1CollectedHeap* g1h, VerifyOption vo) :
 380       AbstractGangTask("Parallel verify task"),
 381       _g1h(g1h),
 382       _vo(vo),
 383       _failures(false),
 384       _hrclaimer(g1h->workers()->active_workers()) {}
 385 
 386   bool failures() {
 387     return _failures;
 388   }
 389 
 390   void work(uint worker_id) {
 391     HandleMark hm;
 392     VerifyRegionClosure blk(true, _vo);
 393     _g1h->heap_region_par_iterate_from_worker_offset(&blk, &_hrclaimer, worker_id);
 394     if (blk.failures()) {
 395       _failures = true;
 396     }
 397   }
 398 };
 399 
 400 void G1HeapVerifier::enable_verification_type(G1VerifyType type) {
 401   // First enable will clear _enabled_verification_types.
 402   if (_enabled_verification_types == G1VerifyAll) {
 403     _enabled_verification_types = type;
 404   } else {
 405     _enabled_verification_types |= type;
 406   }
 407 }
 408 
 409 bool G1HeapVerifier::should_verify(G1VerifyType type) {
 410   return (_enabled_verification_types & type) == type;
 411 }
 412 
 413 void G1HeapVerifier::verify(VerifyOption vo) {
 414   if (!SafepointSynchronize::is_at_safepoint()) {
 415     log_info(gc, verify)("Skipping verification. Not at safepoint.");
 416   }
 417 
 418   assert(Thread::current()->is_VM_thread(),
 419          "Expected to be executed serially by the VM thread at this point");
 420 
 421   log_debug(gc, verify)("Roots");
 422   VerifyRootsClosure rootsCl(vo);
 423   VerifyCLDClosure cldCl(_g1h, &rootsCl);
 424 
 425   // We apply the relevant closures to all the oops in the
 426   // system dictionary, class loader data graph, the string table
 427   // and the nmethods in the code cache.
 428   G1VerifyCodeRootOopClosure codeRootsCl(_g1h, &rootsCl, vo);
 429   G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl);
 430 
 431   {
 432     G1RootProcessor root_processor(_g1h, 1);
 433     root_processor.process_all_roots(&rootsCl,
 434                                      &cldCl,
 435                                      &blobsCl);
 436   }
 437 
 438   bool failures = rootsCl.failures() || codeRootsCl.failures();
 439 
 440   if (!_g1h->g1_policy()->collector_state()->in_full_gc()) {
 441     // If we're verifying during a full GC then the region sets
 442     // will have been torn down at the start of the GC. Therefore
 443     // verifying the region sets will fail. So we only verify
 444     // the region sets when not in a full GC.
 445     log_debug(gc, verify)("HeapRegionSets");
 446     verify_region_sets();
 447   }
 448 
 449   log_debug(gc, verify)("HeapRegions");
 450   if (GCParallelVerificationEnabled && ParallelGCThreads > 1) {
 451 
 452     G1ParVerifyTask task(_g1h, vo);
 453     _g1h->workers()->run_task(&task);
 454     if (task.failures()) {
 455       failures = true;
 456     }
 457 
 458   } else {
 459     VerifyRegionClosure blk(false, vo);
 460     _g1h->heap_region_iterate(&blk);
 461     if (blk.failures()) {
 462       failures = true;
 463     }
 464   }
 465 
 466   if (G1StringDedup::is_enabled()) {
 467     log_debug(gc, verify)("StrDedup");
 468     G1StringDedup::verify();
 469   }
 470 
 471   if (failures) {
 472     log_error(gc, verify)("Heap after failed verification (kind %d):", vo);
 473     // It helps to have the per-region information in the output to
 474     // help us track down what went wrong. This is why we call
 475     // print_extended_on() instead of print_on().
 476     Log(gc, verify) log;
 477     ResourceMark rm;
 478     LogStream ls(log.error());
 479     _g1h->print_extended_on(&ls);
 480   }
 481   guarantee(!failures, "there should not have been any failures");
 482 }
 483 
 484 // Heap region set verification
 485 
 486 class VerifyRegionListsClosure : public HeapRegionClosure {
 487 private:
 488   HeapRegionSet*   _old_set;
 489   HeapRegionSet*   _archive_set;
 490   HeapRegionSet*   _humongous_set;
 491   HeapRegionManager* _hrm;
 492 
 493 public:
 494   uint _old_count;
 495   uint _archive_count;
 496   uint _humongous_count;
 497   uint _free_count;
 498 
 499   VerifyRegionListsClosure(HeapRegionSet* old_set,
 500                            HeapRegionSet* archive_set,
 501                            HeapRegionSet* humongous_set,
 502                            HeapRegionManager* hrm) :
 503     _old_set(old_set), _archive_set(archive_set), _humongous_set(humongous_set), _hrm(hrm),
 504     _old_count(), _archive_count(), _humongous_count(), _free_count(){ }
 505 
 506   bool do_heap_region(HeapRegion* hr) {
 507     if (hr->is_young()) {
 508       // TODO
 509     } else if (hr->is_humongous()) {
 510       assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index());
 511       _humongous_count++;
 512     } else if (hr->is_empty()) {
 513       assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
 514       _free_count++;
 515     } else if (hr->is_archive()) {
 516       assert(hr->containing_set() == _archive_set, "Heap region %u is archive but not in the archive set.", hr->hrm_index());
 517       _archive_count++;
 518     } else if (hr->is_old()) {
 519       assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
 520       _old_count++;
 521     } else {
 522       // There are no other valid region types. Check for one invalid
 523       // one we can identify: pinned without old or humongous set.
 524       assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
 525       ShouldNotReachHere();
 526     }
 527     return false;
 528   }
 529 
 530   void verify_counts(HeapRegionSet* old_set, HeapRegionSet* archive_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
 531     guarantee(old_set->length() == _old_count, "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count);
 532     guarantee(archive_set->length() == _archive_count, "Archive set count mismatch. Expected %u, actual %u.", archive_set->length(), _archive_count);
 533     guarantee(humongous_set->length() == _humongous_count, "Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count);
 534     guarantee(free_list->num_free_regions() == _free_count, "Free list count mismatch. Expected %u, actual %u.", free_list->num_free_regions(), _free_count);
 535   }
 536 };
 537 
 538 void G1HeapVerifier::verify_region_sets() {
 539   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
 540 
 541   // First, check the explicit lists.
 542   _g1h->_hrm.verify();
 543 
 544   // Finally, make sure that the region accounting in the lists is
 545   // consistent with what we see in the heap.
 546 
 547   VerifyRegionListsClosure cl(&_g1h->_old_set, &_g1h->_archive_set, &_g1h->_humongous_set, &_g1h->_hrm);
 548   _g1h->heap_region_iterate(&cl);
 549   cl.verify_counts(&_g1h->_old_set, &_g1h->_archive_set, &_g1h->_humongous_set, &_g1h->_hrm);
 550 }
 551 
 552 void G1HeapVerifier::prepare_for_verify() {
 553   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
 554     _g1h->ensure_parsability(false);
 555   }
 556 }
 557 
 558 double G1HeapVerifier::verify(G1VerifyType type, VerifyOption vo, const char* msg) {
 559   double verify_time_ms = 0.0;
 560 
 561   if (should_verify(type) && _g1h->total_collections() >= VerifyGCStartAt) {
 562     double verify_start = os::elapsedTime();
 563     HandleMark hm;  // Discard invalid handles created during verification
 564     prepare_for_verify();
 565     Universe::verify(vo, msg);
 566     verify_time_ms = (os::elapsedTime() - verify_start) * 1000;
 567   }
 568 
 569   return verify_time_ms;
 570 }
 571 
 572 void G1HeapVerifier::verify_before_gc(G1VerifyType type) {
 573   if (VerifyBeforeGC) {
 574     double verify_time_ms = verify(type, VerifyOption_G1UsePrevMarking, "Before GC");
 575     _g1h->g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms);
 576   }
 577 }
 578 
 579 void G1HeapVerifier::verify_after_gc(G1VerifyType type) {
 580   if (VerifyAfterGC) {
 581     double verify_time_ms = verify(type, VerifyOption_G1UsePrevMarking, "After GC");
 582     _g1h->g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms);
 583   }
 584 }
 585 
 586 
 587 #ifndef PRODUCT
 588 class G1VerifyCardTableCleanup: public HeapRegionClosure {
 589   G1HeapVerifier* _verifier;
 590 public:
 591   G1VerifyCardTableCleanup(G1HeapVerifier* verifier)
 592     : _verifier(verifier) { }
 593   virtual bool do_heap_region(HeapRegion* r) {
 594     if (r->is_survivor()) {
 595       _verifier->verify_dirty_region(r);
 596     } else {
 597       _verifier->verify_not_dirty_region(r);
 598     }
 599     return false;
 600   }
 601 };
 602 
 603 void G1HeapVerifier::verify_card_table_cleanup() {
 604   if (G1VerifyCTCleanup || VerifyAfterGC) {
 605     G1VerifyCardTableCleanup cleanup_verifier(this);
 606     _g1h->heap_region_iterate(&cleanup_verifier);
 607   }
 608 }
 609 
 610 void G1HeapVerifier::verify_not_dirty_region(HeapRegion* hr) {
 611   // All of the region should be clean.
 612   G1CardTable* ct = _g1h->card_table();
 613   MemRegion mr(hr->bottom(), hr->end());
 614   ct->verify_not_dirty_region(mr);
 615 }
 616 
 617 void G1HeapVerifier::verify_dirty_region(HeapRegion* hr) {
 618   // We cannot guarantee that [bottom(),end()] is dirty.  Threads
 619   // dirty allocated blocks as they allocate them. The thread that
 620   // retires each region and replaces it with a new one will do a
 621   // maximal allocation to fill in [pre_dummy_top(),end()] but will
 622   // not dirty that area (one less thing to have to do while holding
 623   // a lock). So we can only verify that [bottom(),pre_dummy_top()]
 624   // is dirty.
 625   G1CardTable* ct = _g1h->card_table();
 626   MemRegion mr(hr->bottom(), hr->pre_dummy_top());
 627   if (hr->is_young()) {
 628     ct->verify_g1_young_region(mr);
 629   } else {
 630     ct->verify_dirty_region(mr);
 631   }
 632 }
 633 
 634 class G1VerifyDirtyYoungListClosure : public HeapRegionClosure {
 635 private:
 636   G1HeapVerifier* _verifier;
 637 public:
 638   G1VerifyDirtyYoungListClosure(G1HeapVerifier* verifier) : HeapRegionClosure(), _verifier(verifier) { }
 639   virtual bool do_heap_region(HeapRegion* r) {
 640     _verifier->verify_dirty_region(r);
 641     return false;
 642   }
 643 };
 644 
 645 void G1HeapVerifier::verify_dirty_young_regions() {
 646   G1VerifyDirtyYoungListClosure cl(this);
 647   _g1h->collection_set()->iterate(&cl);
 648 }
 649 
 650 bool G1HeapVerifier::verify_no_bits_over_tams(const char* bitmap_name, const G1CMBitMap* const bitmap,
 651                                                HeapWord* tams, HeapWord* end) {
 652   guarantee(tams <= end,
 653             "tams: " PTR_FORMAT " end: " PTR_FORMAT, p2i(tams), p2i(end));
 654   HeapWord* result = bitmap->get_next_marked_addr(tams, end);
 655   if (result < end) {
 656     log_error(gc, verify)("## wrong marked address on %s bitmap: " PTR_FORMAT, bitmap_name, p2i(result));
 657     log_error(gc, verify)("## %s tams: " PTR_FORMAT " end: " PTR_FORMAT, bitmap_name, p2i(tams), p2i(end));
 658     return false;
 659   }
 660   return true;
 661 }
 662 
 663 bool G1HeapVerifier::verify_bitmaps(const char* caller, HeapRegion* hr) {
 664   const G1CMBitMap* const prev_bitmap = _g1h->concurrent_mark()->prev_mark_bitmap();
 665   const G1CMBitMap* const next_bitmap = _g1h->concurrent_mark()->next_mark_bitmap();
 666 
 667   HeapWord* ptams  = hr->prev_top_at_mark_start();
 668   HeapWord* ntams  = hr->next_top_at_mark_start();
 669   HeapWord* end    = hr->end();
 670 
 671   bool res_p = verify_no_bits_over_tams("prev", prev_bitmap, ptams, end);
 672 
 673   bool res_n = true;
 674   // We cannot verify the next bitmap while we are about to clear it.
 675   if (!_g1h->collector_state()->clearing_next_bitmap()) {
 676     res_n = verify_no_bits_over_tams("next", next_bitmap, ntams, end);
 677   }
 678   if (!res_p || !res_n) {
 679     log_error(gc, verify)("#### Bitmap verification failed for " HR_FORMAT, HR_FORMAT_PARAMS(hr));
 680     log_error(gc, verify)("#### Caller: %s", caller);
 681     return false;
 682   }
 683   return true;
 684 }
 685 
 686 void G1HeapVerifier::check_bitmaps(const char* caller, HeapRegion* hr) {
 687   if (!G1VerifyBitmaps) {
 688     return;
 689   }
 690 
 691   guarantee(verify_bitmaps(caller, hr), "bitmap verification");
 692 }
 693 
 694 class G1VerifyBitmapClosure : public HeapRegionClosure {
 695 private:
 696   const char* _caller;
 697   G1HeapVerifier* _verifier;
 698   bool _failures;
 699 
 700 public:
 701   G1VerifyBitmapClosure(const char* caller, G1HeapVerifier* verifier) :
 702     _caller(caller), _verifier(verifier), _failures(false) { }
 703 
 704   bool failures() { return _failures; }
 705 
 706   virtual bool do_heap_region(HeapRegion* hr) {
 707     bool result = _verifier->verify_bitmaps(_caller, hr);
 708     if (!result) {
 709       _failures = true;
 710     }
 711     return false;
 712   }
 713 };
 714 
 715 void G1HeapVerifier::check_bitmaps(const char* caller) {
 716   if (!G1VerifyBitmaps) {
 717     return;
 718   }
 719 
 720   G1VerifyBitmapClosure cl(caller, this);
 721   _g1h->heap_region_iterate(&cl);
 722   guarantee(!cl.failures(), "bitmap verification");
 723 }
 724 
 725 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
 726  private:
 727   bool _failures;
 728  public:
 729   G1CheckCSetFastTableClosure() : HeapRegionClosure(), _failures(false) { }
 730 
 731   virtual bool do_heap_region(HeapRegion* hr) {
 732     uint i = hr->hrm_index();
 733     InCSetState cset_state = (InCSetState) G1CollectedHeap::heap()->_in_cset_fast_test.get_by_index(i);
 734     if (hr->is_humongous()) {
 735       if (hr->in_collection_set()) {
 736         log_error(gc, verify)("## humongous region %u in CSet", i);
 737         _failures = true;
 738         return true;
 739       }
 740       if (cset_state.is_in_cset()) {
 741         log_error(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for humongous region %u", cset_state.value(), i);
 742         _failures = true;
 743         return true;
 744       }
 745       if (hr->is_continues_humongous() && cset_state.is_humongous()) {
 746         log_error(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for continues humongous region %u", cset_state.value(), i);
 747         _failures = true;
 748         return true;
 749       }
 750     } else {
 751       if (cset_state.is_humongous()) {
 752         log_error(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for non-humongous region %u", cset_state.value(), i);
 753         _failures = true;
 754         return true;
 755       }
 756       if (hr->in_collection_set() != cset_state.is_in_cset()) {
 757         log_error(gc, verify)("## in CSet %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u",
 758                              hr->in_collection_set(), cset_state.value(), i);
 759         _failures = true;
 760         return true;
 761       }
 762       if (cset_state.is_in_cset()) {
 763         if (hr->is_archive()) {
 764           log_error(gc, verify)("## is_archive in collection set for region %u", i);
 765           _failures = true;
 766           return true;
 767         }
 768         if (hr->is_young() != (cset_state.is_young())) {
 769           log_error(gc, verify)("## is_young %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u",
 770                                hr->is_young(), cset_state.value(), i);
 771           _failures = true;
 772           return true;
 773         }
 774         if (hr->is_old() != (cset_state.is_old())) {
 775           log_error(gc, verify)("## is_old %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u",
 776                                hr->is_old(), cset_state.value(), i);
 777           _failures = true;
 778           return true;
 779         }
 780       }
 781     }
 782     return false;
 783   }
 784 
 785   bool failures() const { return _failures; }
 786 };
 787 
 788 bool G1HeapVerifier::check_cset_fast_test() {
 789   G1CheckCSetFastTableClosure cl;
 790   _g1h->_hrm.iterate(&cl);
 791   return !cl.failures();
 792 }
 793 #endif // PRODUCT