1 /*
   2  * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "code/nmethod.hpp"
  27 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
  28 #include "gc/g1/g1CollectedHeap.inline.hpp"
  29 #include "gc/g1/g1OopClosures.inline.hpp"
  30 #include "gc/g1/heapRegion.inline.hpp"
  31 #include "gc/g1/heapRegionBounds.inline.hpp"
  32 #include "gc/g1/heapRegionManager.inline.hpp"
  33 #include "gc/g1/heapRegionRemSet.hpp"
  34 #include "gc/shared/genOopClosures.inline.hpp"
  35 #include "gc/shared/liveRange.hpp"
  36 #include "gc/shared/space.inline.hpp"
  37 #include "logging/log.hpp"
  38 #include "memory/iterator.hpp"
  39 #include "oops/oop.inline.hpp"
  40 #include "runtime/atomic.inline.hpp"
  41 #include "runtime/orderAccess.inline.hpp"
  42 
  43 int    HeapRegion::LogOfHRGrainBytes = 0;
  44 int    HeapRegion::LogOfHRGrainWords = 0;
  45 size_t HeapRegion::GrainBytes        = 0;
  46 size_t HeapRegion::GrainWords        = 0;
  47 size_t HeapRegion::CardsPerRegion    = 0;
  48 
  49 HeapRegionDCTOC::HeapRegionDCTOC(G1CollectedHeap* g1,
  50                                  HeapRegion* hr,
  51                                  G1ParPushHeapRSClosure* cl,
  52                                  CardTableModRefBS::PrecisionStyle precision) :
  53   DirtyCardToOopClosure(hr, cl, precision, NULL),
  54   _hr(hr), _rs_scan(cl), _g1(g1) { }
  55 
  56 FilterOutOfRegionClosure::FilterOutOfRegionClosure(HeapRegion* r,
  57                                                    OopClosure* oc) :
  58   _r_bottom(r->bottom()), _r_end(r->end()), _oc(oc) { }
  59 
  60 void HeapRegionDCTOC::walk_mem_region(MemRegion mr,
  61                                       HeapWord* bottom,
  62                                       HeapWord* top) {
  63   G1CollectedHeap* g1h = _g1;
  64   size_t oop_size;
  65   HeapWord* cur = bottom;
  66 
  67   // Start filtering what we add to the remembered set. If the object is
  68   // not considered dead, either because it is marked (in the mark bitmap)
  69   // or it was allocated after marking finished, then we add it. Otherwise
  70   // we can safely ignore the object.
  71   if (!g1h->is_obj_dead(oop(cur))) {
  72     oop_size = oop(cur)->oop_iterate_size(_rs_scan, mr);
  73   } else {
  74     oop_size = _hr->block_size(cur);
  75   }
  76 
  77   cur += oop_size;
  78 
  79   if (cur < top) {
  80     oop cur_oop = oop(cur);
  81     oop_size = _hr->block_size(cur);
  82     HeapWord* next_obj = cur + oop_size;
  83     while (next_obj < top) {
  84       // Keep filtering the remembered set.
  85       if (!g1h->is_obj_dead(cur_oop)) {
  86         // Bottom lies entirely below top, so we can call the
  87         // non-memRegion version of oop_iterate below.
  88         cur_oop->oop_iterate(_rs_scan);
  89       }
  90       cur = next_obj;
  91       cur_oop = oop(cur);
  92       oop_size = _hr->block_size(cur);
  93       next_obj = cur + oop_size;
  94     }
  95 
  96     // Last object. Need to do dead-obj filtering here too.
  97     if (!g1h->is_obj_dead(oop(cur))) {
  98       oop(cur)->oop_iterate(_rs_scan, mr);
  99     }
 100   }
 101 }
 102 
 103 size_t HeapRegion::max_region_size() {
 104   return HeapRegionBounds::max_size();
 105 }
 106 
 107 size_t HeapRegion::min_region_size_in_words() {
 108   return HeapRegionBounds::min_size() >> LogHeapWordSize;
 109 }
 110 
 111 void HeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) {
 112   size_t region_size = G1HeapRegionSize;
 113   if (FLAG_IS_DEFAULT(G1HeapRegionSize)) {
 114     size_t average_heap_size = (initial_heap_size + max_heap_size) / 2;
 115     region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(),
 116                        HeapRegionBounds::min_size());
 117   }
 118 
 119   int region_size_log = log2_long((jlong) region_size);
 120   // Recalculate the region size to make sure it's a power of
 121   // 2. This means that region_size is the largest power of 2 that's
 122   // <= what we've calculated so far.
 123   region_size = ((size_t)1 << region_size_log);
 124 
 125   // Now make sure that we don't go over or under our limits.
 126   if (region_size < HeapRegionBounds::min_size()) {
 127     region_size = HeapRegionBounds::min_size();
 128   } else if (region_size > HeapRegionBounds::max_size()) {
 129     region_size = HeapRegionBounds::max_size();
 130   }
 131 
 132   // And recalculate the log.
 133   region_size_log = log2_long((jlong) region_size);
 134 
 135   // Now, set up the globals.
 136   guarantee(LogOfHRGrainBytes == 0, "we should only set it once");
 137   LogOfHRGrainBytes = region_size_log;
 138 
 139   guarantee(LogOfHRGrainWords == 0, "we should only set it once");
 140   LogOfHRGrainWords = LogOfHRGrainBytes - LogHeapWordSize;
 141 
 142   guarantee(GrainBytes == 0, "we should only set it once");
 143   // The cast to int is safe, given that we've bounded region_size by
 144   // MIN_REGION_SIZE and MAX_REGION_SIZE.
 145   GrainBytes = region_size;
 146   log_info(gc, heap)("Heap region size: " SIZE_FORMAT "M", GrainBytes / M);
 147 
 148   guarantee(GrainWords == 0, "we should only set it once");
 149   GrainWords = GrainBytes >> LogHeapWordSize;
 150   guarantee((size_t) 1 << LogOfHRGrainWords == GrainWords, "sanity");
 151 
 152   guarantee(CardsPerRegion == 0, "we should only set it once");
 153   CardsPerRegion = GrainBytes >> CardTableModRefBS::card_shift;
 154 }
 155 
 156 void HeapRegion::reset_after_compaction() {
 157   G1ContiguousSpace::reset_after_compaction();
 158   // After a compaction the mark bitmap is invalid, so we must
 159   // treat all objects as being inside the unmarked area.
 160   zero_marked_bytes();
 161   init_top_at_mark_start();
 162 }
 163 
 164 void HeapRegion::hr_clear(bool par, bool clear_space, bool locked) {
 165   assert(_humongous_start_region == NULL,
 166          "we should have already filtered out humongous regions");
 167   assert(!in_collection_set(),
 168          "Should not clear heap region %u in the collection set", hrm_index());
 169 
 170   set_allocation_context(AllocationContext::system());
 171   set_young_index_in_cset(-1);
 172   uninstall_surv_rate_group();
 173   set_free();
 174   reset_pre_dummy_top();
 175 
 176   if (!par) {
 177     // If this is parallel, this will be done later.
 178     HeapRegionRemSet* hrrs = rem_set();
 179     if (locked) {
 180       hrrs->clear_locked();
 181     } else {
 182       hrrs->clear();
 183     }
 184   }
 185   zero_marked_bytes();
 186 
 187   init_top_at_mark_start();
 188   if (clear_space) clear(SpaceDecorator::Mangle);
 189 }
 190 
 191 void HeapRegion::par_clear() {
 192   assert(used() == 0, "the region should have been already cleared");
 193   assert(capacity() == HeapRegion::GrainBytes, "should be back to normal");
 194   HeapRegionRemSet* hrrs = rem_set();
 195   hrrs->clear();
 196   CardTableModRefBS* ct_bs =
 197     barrier_set_cast<CardTableModRefBS>(G1CollectedHeap::heap()->barrier_set());
 198   ct_bs->clear(MemRegion(bottom(), end()));
 199 }
 200 
 201 void HeapRegion::calc_gc_efficiency() {
 202   // GC efficiency is the ratio of how much space would be
 203   // reclaimed over how long we predict it would take to reclaim it.
 204   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 205   G1CollectorPolicy* g1p = g1h->g1_policy();
 206 
 207   // Retrieve a prediction of the elapsed time for this region for
 208   // a mixed gc because the region will only be evacuated during a
 209   // mixed gc.
 210   double region_elapsed_time_ms =
 211     g1p->predict_region_elapsed_time_ms(this, false /* for_young_gc */);
 212   _gc_efficiency = (double) reclaimable_bytes() / region_elapsed_time_ms;
 213 }
 214 
 215 void HeapRegion::set_starts_humongous(HeapWord* obj_top, size_t fill_size) {
 216   assert(!is_humongous(), "sanity / pre-condition");
 217   assert(top() == bottom(), "should be empty");
 218 
 219   _type.set_starts_humongous();
 220   _humongous_start_region = this;
 221 
 222   _bot_part.set_for_starts_humongous(obj_top, fill_size);
 223 }
 224 
 225 void HeapRegion::set_continues_humongous(HeapRegion* first_hr) {
 226   assert(!is_humongous(), "sanity / pre-condition");
 227   assert(top() == bottom(), "should be empty");
 228   assert(first_hr->is_starts_humongous(), "pre-condition");
 229 
 230   _type.set_continues_humongous();
 231   _humongous_start_region = first_hr;
 232 }
 233 
 234 void HeapRegion::clear_humongous() {
 235   assert(is_humongous(), "pre-condition");
 236 
 237   assert(capacity() == HeapRegion::GrainBytes, "pre-condition");
 238   _humongous_start_region = NULL;
 239 }
 240 
 241 HeapRegion::HeapRegion(uint hrm_index,
 242                        G1BlockOffsetTable* bot,
 243                        MemRegion mr) :
 244     G1ContiguousSpace(bot),
 245     _hrm_index(hrm_index),
 246     _allocation_context(AllocationContext::system()),
 247     _humongous_start_region(NULL),
 248     _next_in_special_set(NULL),
 249     _evacuation_failed(false),
 250     _prev_marked_bytes(0), _next_marked_bytes(0), _gc_efficiency(0.0),
 251     _next_young_region(NULL),
 252     _next_dirty_cards_region(NULL), _next(NULL), _prev(NULL),
 253 #ifdef ASSERT
 254     _containing_set(NULL),
 255 #endif // ASSERT
 256      _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1),
 257     _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0),
 258     _predicted_bytes_to_copy(0)
 259 {
 260   _rem_set = new HeapRegionRemSet(bot, this);
 261 
 262   initialize(mr);
 263 }
 264 
 265 void HeapRegion::initialize(MemRegion mr, bool clear_space, bool mangle_space) {
 266   assert(_rem_set->is_empty(), "Remembered set must be empty");
 267 
 268   G1ContiguousSpace::initialize(mr, clear_space, mangle_space);
 269 
 270   hr_clear(false /*par*/, false /*clear_space*/);
 271   set_top(bottom());
 272   record_timestamp();
 273 }
 274 
 275 CompactibleSpace* HeapRegion::next_compaction_space() const {
 276   return G1CollectedHeap::heap()->next_compaction_region(this);
 277 }
 278 
 279 void HeapRegion::note_self_forwarding_removal_start(bool during_initial_mark,
 280                                                     bool during_conc_mark) {
 281   // We always recreate the prev marking info and we'll explicitly
 282   // mark all objects we find to be self-forwarded on the prev
 283   // bitmap. So all objects need to be below PTAMS.
 284   _prev_marked_bytes = 0;
 285 
 286   if (during_initial_mark) {
 287     // During initial-mark, we'll also explicitly mark all objects
 288     // we find to be self-forwarded on the next bitmap. So all
 289     // objects need to be below NTAMS.
 290     _next_top_at_mark_start = top();
 291     _next_marked_bytes = 0;
 292   } else if (during_conc_mark) {
 293     // During concurrent mark, all objects in the CSet (including
 294     // the ones we find to be self-forwarded) are implicitly live.
 295     // So all objects need to be above NTAMS.
 296     _next_top_at_mark_start = bottom();
 297     _next_marked_bytes = 0;
 298   }
 299 }
 300 
 301 void HeapRegion::note_self_forwarding_removal_end(bool during_initial_mark,
 302                                                   bool during_conc_mark,
 303                                                   size_t marked_bytes) {
 304   assert(marked_bytes <= used(),
 305          "marked: " SIZE_FORMAT " used: " SIZE_FORMAT, marked_bytes, used());
 306   _prev_top_at_mark_start = top();
 307   _prev_marked_bytes = marked_bytes;
 308 }
 309 
 310 HeapWord*
 311 HeapRegion::object_iterate_mem_careful(MemRegion mr,
 312                                                  ObjectClosure* cl) {
 313   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 314   // We used to use "block_start_careful" here.  But we're actually happy
 315   // to update the BOT while we do this...
 316   HeapWord* cur = block_start(mr.start());
 317   mr = mr.intersection(used_region());
 318   if (mr.is_empty()) return NULL;
 319   // Otherwise, find the obj that extends onto mr.start().
 320 
 321   assert(cur <= mr.start()
 322          && (oop(cur)->klass_or_null() == NULL ||
 323              cur + oop(cur)->size() > mr.start()),
 324          "postcondition of block_start");
 325   oop obj;
 326   while (cur < mr.end()) {
 327     obj = oop(cur);
 328     if (obj->klass_or_null() == NULL) {
 329       // Ran into an unparseable point.
 330       return cur;
 331     } else if (!g1h->is_obj_dead(obj)) {
 332       cl->do_object(obj);
 333     }
 334     cur += block_size(cur);
 335   }
 336   return NULL;
 337 }
 338 
 339 HeapWord*
 340 HeapRegion::
 341 oops_on_card_seq_iterate_careful(MemRegion mr,
 342                                  FilterOutOfRegionClosure* cl,
 343                                  bool filter_young,
 344                                  jbyte* card_ptr) {
 345   // Currently, we should only have to clean the card if filter_young
 346   // is true and vice versa.
 347   if (filter_young) {
 348     assert(card_ptr != NULL, "pre-condition");
 349   } else {
 350     assert(card_ptr == NULL, "pre-condition");
 351   }
 352   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 353 
 354   // If we're within a stop-world GC, then we might look at a card in a
 355   // GC alloc region that extends onto a GC LAB, which may not be
 356   // parseable.  Stop such at the "scan_top" of the region.
 357   if (g1h->is_gc_active()) {
 358     mr = mr.intersection(MemRegion(bottom(), scan_top()));
 359   } else {
 360     mr = mr.intersection(used_region());
 361   }
 362   if (mr.is_empty()) return NULL;
 363   // Otherwise, find the obj that extends onto mr.start().
 364 
 365   // The intersection of the incoming mr (for the card) and the
 366   // allocated part of the region is non-empty. This implies that
 367   // we have actually allocated into this region. The code in
 368   // G1CollectedHeap.cpp that allocates a new region sets the
 369   // is_young tag on the region before allocating. Thus we
 370   // safely know if this region is young.
 371   if (is_young() && filter_young) {
 372     return NULL;
 373   }
 374 
 375   assert(!is_young(), "check value of filter_young");
 376 
 377   // We can only clean the card here, after we make the decision that
 378   // the card is not young. And we only clean the card if we have been
 379   // asked to (i.e., card_ptr != NULL).
 380   if (card_ptr != NULL) {
 381     *card_ptr = CardTableModRefBS::clean_card_val();
 382     // We must complete this write before we do any of the reads below.
 383     OrderAccess::storeload();
 384   }
 385 
 386   // Cache the boundaries of the memory region in some const locals
 387   HeapWord* const start = mr.start();
 388   HeapWord* const end = mr.end();
 389 
 390   // We used to use "block_start_careful" here.  But we're actually happy
 391   // to update the BOT while we do this...
 392   HeapWord* cur = block_start(start);
 393   assert(cur <= start, "Postcondition");
 394 
 395   oop obj;
 396 
 397   HeapWord* next = cur;
 398   do {
 399     cur = next;
 400     obj = oop(cur);
 401     if (obj->klass_or_null() == NULL) {
 402       // Ran into an unparseable point.
 403       return cur;
 404     }
 405     // Otherwise...
 406     next = cur + block_size(cur);
 407   } while (next <= start);
 408 
 409   // If we finish the above loop...We have a parseable object that
 410   // begins on or before the start of the memory region, and ends
 411   // inside or spans the entire region.
 412   assert(cur <= start, "Loop postcondition");
 413   assert(obj->klass_or_null() != NULL, "Loop postcondition");
 414 
 415   do {
 416     obj = oop(cur);
 417     assert((cur + block_size(cur)) > (HeapWord*)obj, "Loop invariant");
 418     if (obj->klass_or_null() == NULL) {
 419       // Ran into an unparseable point.
 420       return cur;
 421     }
 422 
 423     // Advance the current pointer. "obj" still points to the object to iterate.
 424     cur = cur + block_size(cur);
 425 
 426     if (!g1h->is_obj_dead(obj)) {
 427       // Non-objArrays are sometimes marked imprecise at the object start. We
 428       // always need to iterate over them in full.
 429       // We only iterate over object arrays in full if they are completely contained
 430       // in the memory region.
 431       if (!obj->is_objArray() || (((HeapWord*)obj) >= start && cur <= end)) {
 432         obj->oop_iterate(cl);
 433       } else {
 434         obj->oop_iterate(cl, mr);
 435       }
 436     }
 437   } while (cur < end);
 438 
 439   return NULL;
 440 }
 441 
 442 // Code roots support
 443 
 444 void HeapRegion::add_strong_code_root(nmethod* nm) {
 445   HeapRegionRemSet* hrrs = rem_set();
 446   hrrs->add_strong_code_root(nm);
 447 }
 448 
 449 void HeapRegion::add_strong_code_root_locked(nmethod* nm) {
 450   assert_locked_or_safepoint(CodeCache_lock);
 451   HeapRegionRemSet* hrrs = rem_set();
 452   hrrs->add_strong_code_root_locked(nm);
 453 }
 454 
 455 void HeapRegion::remove_strong_code_root(nmethod* nm) {
 456   HeapRegionRemSet* hrrs = rem_set();
 457   hrrs->remove_strong_code_root(nm);
 458 }
 459 
 460 void HeapRegion::strong_code_roots_do(CodeBlobClosure* blk) const {
 461   HeapRegionRemSet* hrrs = rem_set();
 462   hrrs->strong_code_roots_do(blk);
 463 }
 464 
 465 class VerifyStrongCodeRootOopClosure: public OopClosure {
 466   const HeapRegion* _hr;
 467   nmethod* _nm;
 468   bool _failures;
 469   bool _has_oops_in_region;
 470 
 471   template <class T> void do_oop_work(T* p) {
 472     T heap_oop = oopDesc::load_heap_oop(p);
 473     if (!oopDesc::is_null(heap_oop)) {
 474       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
 475 
 476       // Note: not all the oops embedded in the nmethod are in the
 477       // current region. We only look at those which are.
 478       if (_hr->is_in(obj)) {
 479         // Object is in the region. Check that its less than top
 480         if (_hr->top() <= (HeapWord*)obj) {
 481           // Object is above top
 482           log_error(gc, verify)("Object " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ") is above top " PTR_FORMAT,
 483                                p2i(obj), p2i(_hr->bottom()), p2i(_hr->end()), p2i(_hr->top()));
 484           _failures = true;
 485           return;
 486         }
 487         // Nmethod has at least one oop in the current region
 488         _has_oops_in_region = true;
 489       }
 490     }
 491   }
 492 
 493 public:
 494   VerifyStrongCodeRootOopClosure(const HeapRegion* hr, nmethod* nm):
 495     _hr(hr), _failures(false), _has_oops_in_region(false) {}
 496 
 497   void do_oop(narrowOop* p) { do_oop_work(p); }
 498   void do_oop(oop* p)       { do_oop_work(p); }
 499 
 500   bool failures()           { return _failures; }
 501   bool has_oops_in_region() { return _has_oops_in_region; }
 502 };
 503 
 504 class VerifyStrongCodeRootCodeBlobClosure: public CodeBlobClosure {
 505   const HeapRegion* _hr;
 506   bool _failures;
 507 public:
 508   VerifyStrongCodeRootCodeBlobClosure(const HeapRegion* hr) :
 509     _hr(hr), _failures(false) {}
 510 
 511   void do_code_blob(CodeBlob* cb) {
 512     nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null();
 513     if (nm != NULL) {
 514       // Verify that the nemthod is live
 515       if (!nm->is_alive()) {
 516         log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has dead nmethod " PTR_FORMAT " in its strong code roots",
 517                               p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
 518         _failures = true;
 519       } else {
 520         VerifyStrongCodeRootOopClosure oop_cl(_hr, nm);
 521         nm->oops_do(&oop_cl);
 522         if (!oop_cl.has_oops_in_region()) {
 523           log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has nmethod " PTR_FORMAT " in its strong code roots with no pointers into region",
 524                                 p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
 525           _failures = true;
 526         } else if (oop_cl.failures()) {
 527           log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has other failures for nmethod " PTR_FORMAT,
 528                                 p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
 529           _failures = true;
 530         }
 531       }
 532     }
 533   }
 534 
 535   bool failures()       { return _failures; }
 536 };
 537 
 538 void HeapRegion::verify_strong_code_roots(VerifyOption vo, bool* failures) const {
 539   if (!G1VerifyHeapRegionCodeRoots) {
 540     // We're not verifying code roots.
 541     return;
 542   }
 543   if (vo == VerifyOption_G1UseMarkWord) {
 544     // Marking verification during a full GC is performed after class
 545     // unloading, code cache unloading, etc so the strong code roots
 546     // attached to each heap region are in an inconsistent state. They won't
 547     // be consistent until the strong code roots are rebuilt after the
 548     // actual GC. Skip verifying the strong code roots in this particular
 549     // time.
 550     assert(VerifyDuringGC, "only way to get here");
 551     return;
 552   }
 553 
 554   HeapRegionRemSet* hrrs = rem_set();
 555   size_t strong_code_roots_length = hrrs->strong_code_roots_list_length();
 556 
 557   // if this region is empty then there should be no entries
 558   // on its strong code root list
 559   if (is_empty()) {
 560     if (strong_code_roots_length > 0) {
 561       log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] is empty but has " SIZE_FORMAT " code root entries",
 562                             p2i(bottom()), p2i(end()), strong_code_roots_length);
 563       *failures = true;
 564     }
 565     return;
 566   }
 567 
 568   if (is_continues_humongous()) {
 569     if (strong_code_roots_length > 0) {
 570       log_error(gc, verify)("region " HR_FORMAT " is a continuation of a humongous region but has " SIZE_FORMAT " code root entries",
 571                             HR_FORMAT_PARAMS(this), strong_code_roots_length);
 572       *failures = true;
 573     }
 574     return;
 575   }
 576 
 577   VerifyStrongCodeRootCodeBlobClosure cb_cl(this);
 578   strong_code_roots_do(&cb_cl);
 579 
 580   if (cb_cl.failures()) {
 581     *failures = true;
 582   }
 583 }
 584 
 585 void HeapRegion::print() const { print_on(tty); }
 586 void HeapRegion::print_on(outputStream* st) const {
 587   st->print("|%4u", this->_hrm_index);
 588   st->print("|" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT,
 589             p2i(bottom()), p2i(top()), p2i(end()));
 590   st->print("|%3d%%", (int) ((double) used() * 100 / capacity()));
 591   st->print("|%2s", get_short_type_str());
 592   if (in_collection_set()) {
 593     st->print("|CS");
 594   } else {
 595     st->print("|  ");
 596   }
 597   st->print("|TS%3u", _gc_time_stamp);
 598   st->print("|AC%3u", allocation_context());
 599   st->print_cr("|TAMS " PTR_FORMAT ", " PTR_FORMAT "|",
 600                p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start()));
 601 }
 602 
 603 class G1VerificationClosure : public OopClosure {
 604 protected:
 605   G1CollectedHeap* _g1h;
 606   CardTableModRefBS* _bs;
 607   oop _containing_obj;
 608   bool _failures;
 609   int _n_failures;
 610   VerifyOption _vo;
 611 public:
 612   // _vo == UsePrevMarking -> use "prev" marking information,
 613   // _vo == UseNextMarking -> use "next" marking information,
 614   // _vo == UseMarkWord    -> use mark word from object header.
 615   G1VerificationClosure(G1CollectedHeap* g1h, VerifyOption vo) :
 616     _g1h(g1h), _bs(barrier_set_cast<CardTableModRefBS>(g1h->barrier_set())),
 617     _containing_obj(NULL), _failures(false), _n_failures(0), _vo(vo) {
 618   }
 619 
 620   void set_containing_obj(oop obj) {
 621     _containing_obj = obj;
 622   }
 623 
 624   bool failures() { return _failures; }
 625   int n_failures() { return _n_failures; }
 626 
 627   void print_object(outputStream* out, oop obj) {
 628 #ifdef PRODUCT
 629     Klass* k = obj->klass();
 630     const char* class_name = k->external_name();
 631     out->print_cr("class name %s", class_name);
 632 #else // PRODUCT
 633     obj->print_on(out);
 634 #endif // PRODUCT
 635   }
 636 };
 637 
 638 class VerifyLiveClosure : public G1VerificationClosure {
 639 public:
 640   VerifyLiveClosure(G1CollectedHeap* g1h, VerifyOption vo) : G1VerificationClosure(g1h, vo) {}
 641   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
 642   virtual void do_oop(oop* p) { do_oop_work(p); }
 643 
 644   template <class T>
 645   void do_oop_work(T* p) {
 646     assert(_containing_obj != NULL, "Precondition");
 647     assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo),
 648       "Precondition");
 649     verify_liveness(p);
 650   }
 651 
 652   template <class T>
 653   void verify_liveness(T* p) {
 654     T heap_oop = oopDesc::load_heap_oop(p);
 655     LogHandle(gc, verify) log;
 656     if (!oopDesc::is_null(heap_oop)) {
 657       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
 658       bool failed = false;
 659       if (!_g1h->is_in_closed_subset(obj) || _g1h->is_obj_dead_cond(obj, _vo)) {
 660         MutexLockerEx x(ParGCRareEvent_lock,
 661           Mutex::_no_safepoint_check_flag);
 662 
 663         if (!_failures) {
 664           log.error("----------");
 665         }
 666         ResourceMark rm;
 667         if (!_g1h->is_in_closed_subset(obj)) {
 668           HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
 669           log.error("Field " PTR_FORMAT " of live obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
 670             p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
 671           print_object(log.error_stream(), _containing_obj);
 672           log.error("points to obj " PTR_FORMAT " not in the heap", p2i(obj));
 673         } else {
 674           HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
 675           HeapRegion* to = _g1h->heap_region_containing((HeapWord*)obj);
 676           log.error("Field " PTR_FORMAT " of live obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
 677             p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
 678           print_object(log.error_stream(), _containing_obj);
 679           log.error("points to dead obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
 680             p2i(obj), p2i(to->bottom()), p2i(to->end()));
 681           print_object(log.error_stream(), obj);
 682         }
 683         log.error("----------");
 684         _failures = true;
 685         failed = true;
 686         _n_failures++;
 687       }
 688     }
 689   }
 690 };
 691 
 692 class VerifyRemSetClosure : public G1VerificationClosure {
 693 public:
 694   VerifyRemSetClosure(G1CollectedHeap* g1h, VerifyOption vo) : G1VerificationClosure(g1h, vo) {}
 695   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
 696   virtual void do_oop(oop* p) { do_oop_work(p); }
 697 
 698   template <class T>
 699   void do_oop_work(T* p) {
 700     assert(_containing_obj != NULL, "Precondition");
 701     assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo),
 702       "Precondition");
 703     verify_remembered_set(p);
 704   }
 705 
 706   template <class T>
 707   void verify_remembered_set(T* p) {
 708     T heap_oop = oopDesc::load_heap_oop(p);
 709     LogHandle(gc, verify) log;
 710     if (!oopDesc::is_null(heap_oop)) {
 711       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
 712       bool failed = false;
 713       HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
 714       HeapRegion* to = _g1h->heap_region_containing(obj);
 715       if (from != NULL && to != NULL &&
 716         from != to &&
 717         !to->is_pinned()) {
 718         jbyte cv_obj = *_bs->byte_for_const(_containing_obj);
 719         jbyte cv_field = *_bs->byte_for_const(p);
 720         const jbyte dirty = CardTableModRefBS::dirty_card_val();
 721 
 722         bool is_bad = !(from->is_young()
 723           || to->rem_set()->contains_reference(p)
 724           || !G1HRRSFlushLogBuffersOnVerify && // buffers were not flushed
 725           (_containing_obj->is_objArray() ?
 726           cv_field == dirty
 727           : cv_obj == dirty || cv_field == dirty));
 728         if (is_bad) {
 729           MutexLockerEx x(ParGCRareEvent_lock,
 730             Mutex::_no_safepoint_check_flag);
 731 
 732           if (!_failures) {
 733             log.error("----------");
 734           }
 735           log.error("Missing rem set entry:");
 736           log.error("Field " PTR_FORMAT " of obj " PTR_FORMAT ", in region " HR_FORMAT,
 737             p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
 738           ResourceMark rm;
 739           _containing_obj->print_on(log.error_stream());
 740           log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT, p2i(obj), HR_FORMAT_PARAMS(to));
 741           obj->print_on(log.error_stream());
 742           log.error("Obj head CTE = %d, field CTE = %d.", cv_obj, cv_field);
 743           log.error("----------");
 744           _failures = true;
 745           if (!failed) _n_failures++;
 746         }
 747       }
 748     }
 749   }
 750 };
 751 
 752 // This really ought to be commoned up into OffsetTableContigSpace somehow.
 753 // We would need a mechanism to make that code skip dead objects.
 754 
 755 void HeapRegion::verify(VerifyOption vo,
 756                         bool* failures) const {
 757   G1CollectedHeap* g1 = G1CollectedHeap::heap();
 758   *failures = false;
 759   HeapWord* p = bottom();
 760   HeapWord* prev_p = NULL;
 761   VerifyLiveClosure vl_cl(g1, vo);
 762   VerifyRemSetClosure vr_cl(g1, vo);
 763   bool is_region_humongous = is_humongous();
 764   size_t object_num = 0;
 765   while (p < top()) {
 766     oop obj = oop(p);
 767     size_t obj_size = block_size(p);
 768     object_num += 1;
 769 
 770     if (!g1->is_obj_dead_cond(obj, this, vo)) {
 771       if (obj->is_oop()) {
 772         Klass* klass = obj->klass();
 773         bool is_metaspace_object = Metaspace::contains(klass) ||
 774                                    (vo == VerifyOption_G1UsePrevMarking &&
 775                                    ClassLoaderDataGraph::unload_list_contains(klass));
 776         if (!is_metaspace_object) {
 777           log_error(gc, verify)("klass " PTR_FORMAT " of object " PTR_FORMAT " "
 778                                 "not metadata", p2i(klass), p2i(obj));
 779           *failures = true;
 780           return;
 781         } else if (!klass->is_klass()) {
 782           log_error(gc, verify)("klass " PTR_FORMAT " of object " PTR_FORMAT " "
 783                                 "not a klass", p2i(klass), p2i(obj));
 784           *failures = true;
 785           return;
 786         } else {
 787           vl_cl.set_containing_obj(obj);
 788           if (!g1->collector_state()->full_collection() || G1VerifyRSetsDuringFullGC) {
 789             // verify liveness and rem_set
 790             vr_cl.set_containing_obj(obj);
 791             G1Mux2Closure mux(&vl_cl, &vr_cl);
 792             obj->oop_iterate_no_header(&mux);
 793 
 794             if (vr_cl.failures()) {
 795               *failures = true;
 796             }
 797             if (G1MaxVerifyFailures >= 0 &&
 798               vr_cl.n_failures() >= G1MaxVerifyFailures) {
 799               return;
 800             }
 801           } else {
 802             // verify only liveness
 803             obj->oop_iterate_no_header(&vl_cl);
 804           }
 805           if (vl_cl.failures()) {
 806             *failures = true;
 807           }
 808           if (G1MaxVerifyFailures >= 0 &&
 809               vl_cl.n_failures() >= G1MaxVerifyFailures) {
 810             return;
 811           }
 812         }
 813       } else {
 814         log_error(gc, verify)(PTR_FORMAT " not an oop", p2i(obj));
 815         *failures = true;
 816         return;
 817       }
 818     }
 819     prev_p = p;
 820     p += obj_size;
 821   }
 822 
 823   if (!is_young() && !is_empty()) {
 824     _bot_part.verify();
 825   }
 826 
 827   if (is_region_humongous) {
 828     oop obj = oop(this->humongous_start_region()->bottom());
 829     if ((HeapWord*)obj > bottom() || (HeapWord*)obj + obj->size() < bottom()) {
 830       log_error(gc, verify)("this humongous region is not part of its' humongous object " PTR_FORMAT, p2i(obj));
 831     }
 832   }
 833 
 834   if (!is_region_humongous && p != top()) {
 835     log_error(gc, verify)("end of last object " PTR_FORMAT " "
 836                           "does not match top " PTR_FORMAT, p2i(p), p2i(top()));
 837     *failures = true;
 838     return;
 839   }
 840 
 841   HeapWord* the_end = end();
 842   // Do some extra BOT consistency checking for addresses in the
 843   // range [top, end). BOT look-ups in this range should yield
 844   // top. No point in doing that if top == end (there's nothing there).
 845   if (p < the_end) {
 846     // Look up top
 847     HeapWord* addr_1 = p;
 848     HeapWord* b_start_1 = _bot_part.block_start_const(addr_1);
 849     if (b_start_1 != p) {
 850       log_error(gc, verify)("BOT look up for top: " PTR_FORMAT " "
 851                             " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
 852                             p2i(addr_1), p2i(b_start_1), p2i(p));
 853       *failures = true;
 854       return;
 855     }
 856 
 857     // Look up top + 1
 858     HeapWord* addr_2 = p + 1;
 859     if (addr_2 < the_end) {
 860       HeapWord* b_start_2 = _bot_part.block_start_const(addr_2);
 861       if (b_start_2 != p) {
 862         log_error(gc, verify)("BOT look up for top + 1: " PTR_FORMAT " "
 863                               " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
 864                               p2i(addr_2), p2i(b_start_2), p2i(p));
 865         *failures = true;
 866         return;
 867       }
 868     }
 869 
 870     // Look up an address between top and end
 871     size_t diff = pointer_delta(the_end, p) / 2;
 872     HeapWord* addr_3 = p + diff;
 873     if (addr_3 < the_end) {
 874       HeapWord* b_start_3 = _bot_part.block_start_const(addr_3);
 875       if (b_start_3 != p) {
 876         log_error(gc, verify)("BOT look up for top + diff: " PTR_FORMAT " "
 877                               " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
 878                               p2i(addr_3), p2i(b_start_3), p2i(p));
 879         *failures = true;
 880         return;
 881       }
 882     }
 883 
 884     // Look up end - 1
 885     HeapWord* addr_4 = the_end - 1;
 886     HeapWord* b_start_4 = _bot_part.block_start_const(addr_4);
 887     if (b_start_4 != p) {
 888       log_error(gc, verify)("BOT look up for end - 1: " PTR_FORMAT " "
 889                             " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
 890                             p2i(addr_4), p2i(b_start_4), p2i(p));
 891       *failures = true;
 892       return;
 893     }
 894   }
 895 
 896   verify_strong_code_roots(vo, failures);
 897 }
 898 
 899 void HeapRegion::verify() const {
 900   bool dummy = false;
 901   verify(VerifyOption_G1UsePrevMarking, /* failures */ &dummy);
 902 }
 903 
 904 void HeapRegion::verify_rem_set(VerifyOption vo, bool* failures) const {
 905   G1CollectedHeap* g1 = G1CollectedHeap::heap();
 906   *failures = false;
 907   HeapWord* p = bottom();
 908   HeapWord* prev_p = NULL;
 909   VerifyRemSetClosure vr_cl(g1, vo);
 910   while (p < top()) {
 911     oop obj = oop(p);
 912     size_t obj_size = block_size(p);
 913 
 914     if (!g1->is_obj_dead_cond(obj, this, vo)) {
 915       if (obj->is_oop()) {
 916         vr_cl.set_containing_obj(obj);
 917         obj->oop_iterate_no_header(&vr_cl);
 918 
 919         if (vr_cl.failures()) {
 920           *failures = true;
 921         }
 922         if (G1MaxVerifyFailures >= 0 &&
 923           vr_cl.n_failures() >= G1MaxVerifyFailures) {
 924           return;
 925         }
 926       } else {
 927         log_error(gc, verify)(PTR_FORMAT " not an oop", p2i(obj));
 928         *failures = true;
 929         return;
 930       }
 931     }
 932 
 933     prev_p = p;
 934     p += obj_size;
 935   }
 936 }
 937 
 938 void HeapRegion::verify_rem_set() const {
 939   bool failures = false;
 940   verify_rem_set(VerifyOption_G1UsePrevMarking, &failures);
 941   guarantee(!failures, "HeapRegion RemSet verification failed");
 942 }
 943 
 944 void HeapRegion::prepare_for_compaction(CompactPoint* cp) {
 945   scan_and_forward(this, cp);
 946 }
 947 
 948 // G1OffsetTableContigSpace code; copied from space.cpp.  Hope this can go
 949 // away eventually.
 950 
 951 void G1ContiguousSpace::clear(bool mangle_space) {
 952   set_top(bottom());
 953   _scan_top = bottom();
 954   CompactibleSpace::clear(mangle_space);
 955   reset_bot();
 956 }
 957 
 958 #ifndef PRODUCT
 959 void G1ContiguousSpace::mangle_unused_area() {
 960   mangle_unused_area_complete();
 961 }
 962 
 963 void G1ContiguousSpace::mangle_unused_area_complete() {
 964   SpaceMangler::mangle_region(MemRegion(top(), end()));
 965 }
 966 #endif
 967 
 968 void G1ContiguousSpace::print() const {
 969   print_short();
 970   tty->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", "
 971                 INTPTR_FORMAT ", " INTPTR_FORMAT ")",
 972                 p2i(bottom()), p2i(top()), p2i(_bot_part.threshold()), p2i(end()));
 973 }
 974 
 975 HeapWord* G1ContiguousSpace::initialize_threshold() {
 976   return _bot_part.initialize_threshold();
 977 }
 978 
 979 HeapWord* G1ContiguousSpace::cross_threshold(HeapWord* start,
 980                                                     HeapWord* end) {
 981   _bot_part.alloc_block(start, end);
 982   return _bot_part.threshold();
 983 }
 984 
 985 HeapWord* G1ContiguousSpace::scan_top() const {
 986   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 987   HeapWord* local_top = top();
 988   OrderAccess::loadload();
 989   const unsigned local_time_stamp = _gc_time_stamp;
 990   assert(local_time_stamp <= g1h->get_gc_time_stamp(), "invariant");
 991   if (local_time_stamp < g1h->get_gc_time_stamp()) {
 992     return local_top;
 993   } else {
 994     return _scan_top;
 995   }
 996 }
 997 
 998 void G1ContiguousSpace::record_timestamp() {
 999   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1000   unsigned curr_gc_time_stamp = g1h->get_gc_time_stamp();
1001 
1002   if (_gc_time_stamp < curr_gc_time_stamp) {
1003     // Setting the time stamp here tells concurrent readers to look at
1004     // scan_top to know the maximum allowed address to look at.
1005 
1006     // scan_top should be bottom for all regions except for the
1007     // retained old alloc region which should have scan_top == top
1008     HeapWord* st = _scan_top;
1009     guarantee(st == _bottom || st == _top, "invariant");
1010 
1011     _gc_time_stamp = curr_gc_time_stamp;
1012   }
1013 }
1014 
1015 void G1ContiguousSpace::record_retained_region() {
1016   // scan_top is the maximum address where it's safe for the next gc to
1017   // scan this region.
1018   _scan_top = top();
1019 }
1020 
1021 void G1ContiguousSpace::safe_object_iterate(ObjectClosure* blk) {
1022   object_iterate(blk);
1023 }
1024 
1025 void G1ContiguousSpace::object_iterate(ObjectClosure* blk) {
1026   HeapWord* p = bottom();
1027   while (p < top()) {
1028     if (block_is_obj(p)) {
1029       blk->do_object(oop(p));
1030     }
1031     p += block_size(p);
1032   }
1033 }
1034 
1035 G1ContiguousSpace::G1ContiguousSpace(G1BlockOffsetTable* bot) :
1036   _bot_part(bot, this),
1037   _par_alloc_lock(Mutex::leaf, "OffsetTableContigSpace par alloc lock", true),
1038   _gc_time_stamp(0)
1039 {
1040 }
1041 
1042 void G1ContiguousSpace::initialize(MemRegion mr, bool clear_space, bool mangle_space) {
1043   CompactibleSpace::initialize(mr, clear_space, mangle_space);
1044   _top = bottom();
1045   _scan_top = bottom();
1046   set_saved_mark_word(NULL);
1047   reset_bot();
1048 }
1049