1 /*
   2  * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "code/nmethod.hpp"
  27 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
  28 #include "gc/g1/g1CollectedHeap.inline.hpp"
  29 #include "gc/g1/g1HeapRegionTraceType.hpp"
  30 #include "gc/g1/g1OopClosures.inline.hpp"
  31 #include "gc/g1/heapRegion.inline.hpp"
  32 #include "gc/g1/heapRegionBounds.inline.hpp"
  33 #include "gc/g1/heapRegionManager.inline.hpp"
  34 #include "gc/g1/heapRegionRemSet.hpp"
  35 #include "gc/g1/heapRegionTracer.hpp"
  36 #include "gc/shared/genOopClosures.inline.hpp"
  37 #include "gc/shared/space.inline.hpp"
  38 #include "logging/log.hpp"
  39 #include "memory/iterator.hpp"
  40 #include "memory/resourceArea.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "runtime/atomic.hpp"
  43 #include "runtime/orderAccess.inline.hpp"
  44 
  45 int    HeapRegion::LogOfHRGrainBytes = 0;
  46 int    HeapRegion::LogOfHRGrainWords = 0;
  47 size_t HeapRegion::GrainBytes        = 0;
  48 size_t HeapRegion::GrainWords        = 0;
  49 size_t HeapRegion::CardsPerRegion    = 0;
  50 
  51 HeapRegionDCTOC::HeapRegionDCTOC(G1CollectedHeap* g1,
  52                                  HeapRegion* hr,
  53                                  G1ParPushHeapRSClosure* cl,
  54                                  CardTable::PrecisionStyle precision) :
  55   DirtyCardToOopClosure(hr, cl, precision, NULL),
  56   _hr(hr), _rs_scan(cl), _g1(g1) { }
  57 
  58 void HeapRegionDCTOC::walk_mem_region(MemRegion mr,
  59                                       HeapWord* bottom,
  60                                       HeapWord* top) {
  61   G1CollectedHeap* g1h = _g1;
  62   size_t oop_size;
  63   HeapWord* cur = bottom;
  64 
  65   // Start filtering what we add to the remembered set. If the object is
  66   // not considered dead, either because it is marked (in the mark bitmap)
  67   // or it was allocated after marking finished, then we add it. Otherwise
  68   // we can safely ignore the object.
  69   if (!g1h->is_obj_dead(oop(cur))) {
  70     oop_size = oop(cur)->oop_iterate_size(_rs_scan, mr);
  71   } else {
  72     oop_size = _hr->block_size(cur);
  73   }
  74 
  75   cur += oop_size;
  76 
  77   if (cur < top) {
  78     oop cur_oop = oop(cur);
  79     oop_size = _hr->block_size(cur);
  80     HeapWord* next_obj = cur + oop_size;
  81     while (next_obj < top) {
  82       // Keep filtering the remembered set.
  83       if (!g1h->is_obj_dead(cur_oop)) {
  84         // Bottom lies entirely below top, so we can call the
  85         // non-memRegion version of oop_iterate below.
  86         cur_oop->oop_iterate(_rs_scan);
  87       }
  88       cur = next_obj;
  89       cur_oop = oop(cur);
  90       oop_size = _hr->block_size(cur);
  91       next_obj = cur + oop_size;
  92     }
  93 
  94     // Last object. Need to do dead-obj filtering here too.
  95     if (!g1h->is_obj_dead(oop(cur))) {
  96       oop(cur)->oop_iterate(_rs_scan, mr);
  97     }
  98   }
  99 }
 100 
 101 size_t HeapRegion::max_region_size() {
 102   return HeapRegionBounds::max_size();
 103 }
 104 
 105 size_t HeapRegion::min_region_size_in_words() {
 106   return HeapRegionBounds::min_size() >> LogHeapWordSize;
 107 }
 108 
 109 void HeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) {
 110   size_t region_size = G1HeapRegionSize;
 111   if (FLAG_IS_DEFAULT(G1HeapRegionSize)) {
 112     size_t average_heap_size = (initial_heap_size + max_heap_size) / 2;
 113     region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(),
 114                        HeapRegionBounds::min_size());
 115   }
 116 
 117   int region_size_log = log2_long((jlong) region_size);
 118   // Recalculate the region size to make sure it's a power of
 119   // 2. This means that region_size is the largest power of 2 that's
 120   // <= what we've calculated so far.
 121   region_size = ((size_t)1 << region_size_log);
 122 
 123   // Now make sure that we don't go over or under our limits.
 124   if (region_size < HeapRegionBounds::min_size()) {
 125     region_size = HeapRegionBounds::min_size();
 126   } else if (region_size > HeapRegionBounds::max_size()) {
 127     region_size = HeapRegionBounds::max_size();
 128   }
 129 
 130   // And recalculate the log.
 131   region_size_log = log2_long((jlong) region_size);
 132 
 133   // Now, set up the globals.
 134   guarantee(LogOfHRGrainBytes == 0, "we should only set it once");
 135   LogOfHRGrainBytes = region_size_log;
 136 
 137   guarantee(LogOfHRGrainWords == 0, "we should only set it once");
 138   LogOfHRGrainWords = LogOfHRGrainBytes - LogHeapWordSize;
 139 
 140   guarantee(GrainBytes == 0, "we should only set it once");
 141   // The cast to int is safe, given that we've bounded region_size by
 142   // MIN_REGION_SIZE and MAX_REGION_SIZE.
 143   GrainBytes = region_size;
 144   log_info(gc, heap)("Heap region size: " SIZE_FORMAT "M", GrainBytes / M);
 145 
 146   guarantee(GrainWords == 0, "we should only set it once");
 147   GrainWords = GrainBytes >> LogHeapWordSize;
 148   guarantee((size_t) 1 << LogOfHRGrainWords == GrainWords, "sanity");
 149 
 150   guarantee(CardsPerRegion == 0, "we should only set it once");
 151   CardsPerRegion = GrainBytes >> G1CardTable::card_shift;
 152 
 153   if (G1HeapRegionSize != GrainBytes) {
 154     FLAG_SET_ERGO(size_t, G1HeapRegionSize, GrainBytes);
 155   }
 156 }
 157 
 158 void HeapRegion::reset_after_compaction() {
 159   G1ContiguousSpace::reset_after_compaction();
 160   // After a compaction the mark bitmap is invalid, so we must
 161   // treat all objects as being inside the unmarked area.
 162   zero_marked_bytes();
 163   init_top_at_mark_start();
 164 }
 165 
 166 void HeapRegion::hr_clear(bool keep_remset, bool clear_space, bool locked) {
 167   assert(_humongous_start_region == NULL,
 168          "we should have already filtered out humongous regions");
 169   assert(!in_collection_set(),
 170          "Should not clear heap region %u in the collection set", hrm_index());
 171 
 172   set_allocation_context(AllocationContext::system());
 173   set_young_index_in_cset(-1);
 174   uninstall_surv_rate_group();
 175   set_free();
 176   reset_pre_dummy_top();
 177 
 178   if (!keep_remset) {
 179     if (locked) {
 180       rem_set()->clear_locked();
 181     } else {
 182       rem_set()->clear();
 183     }
 184   }
 185 
 186   zero_marked_bytes();
 187 
 188   init_top_at_mark_start();
 189   _gc_time_stamp = G1CollectedHeap::heap()->get_gc_time_stamp();
 190   if (clear_space) clear(SpaceDecorator::Mangle);
 191 }
 192 
 193 void HeapRegion::par_clear() {
 194   assert(used() == 0, "the region should have been already cleared");
 195   assert(capacity() == HeapRegion::GrainBytes, "should be back to normal");
 196   HeapRegionRemSet* hrrs = rem_set();
 197   hrrs->clear();
 198   G1CardTable* ct = G1CollectedHeap::heap()->g1_card_table();
 199   ct->clear(MemRegion(bottom(), end()));
 200 }
 201 
 202 void HeapRegion::calc_gc_efficiency() {
 203   // GC efficiency is the ratio of how much space would be
 204   // reclaimed over how long we predict it would take to reclaim it.
 205   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 206   G1Policy* g1p = g1h->g1_policy();
 207 
 208   // Retrieve a prediction of the elapsed time for this region for
 209   // a mixed gc because the region will only be evacuated during a
 210   // mixed gc.
 211   double region_elapsed_time_ms =
 212     g1p->predict_region_elapsed_time_ms(this, false /* for_young_gc */);
 213   _gc_efficiency = (double) reclaimable_bytes() / region_elapsed_time_ms;
 214 }
 215 
 216 void HeapRegion::set_free() {
 217   report_region_type_change(G1HeapRegionTraceType::Free);
 218   _type.set_free();
 219 }
 220 
 221 void HeapRegion::set_eden() {
 222   report_region_type_change(G1HeapRegionTraceType::Eden);
 223   _type.set_eden();
 224 }
 225 
 226 void HeapRegion::set_eden_pre_gc() {
 227   report_region_type_change(G1HeapRegionTraceType::Eden);
 228   _type.set_eden_pre_gc();
 229 }
 230 
 231 void HeapRegion::set_survivor() {
 232   report_region_type_change(G1HeapRegionTraceType::Survivor);
 233   _type.set_survivor();
 234 }
 235 
 236 void HeapRegion::set_old() {
 237   report_region_type_change(G1HeapRegionTraceType::Old);
 238   _type.set_old();
 239 }
 240 
 241 void HeapRegion::set_archive() {
 242   report_region_type_change(G1HeapRegionTraceType::Archive);
 243   _type.set_archive();
 244 }
 245 
 246 void HeapRegion::set_starts_humongous(HeapWord* obj_top, size_t fill_size) {
 247   assert(!is_humongous(), "sanity / pre-condition");
 248   assert(top() == bottom(), "should be empty");
 249 
 250   report_region_type_change(G1HeapRegionTraceType::StartsHumongous);
 251   _type.set_starts_humongous();
 252   _humongous_start_region = this;
 253 
 254   _bot_part.set_for_starts_humongous(obj_top, fill_size);
 255 }
 256 
 257 void HeapRegion::set_continues_humongous(HeapRegion* first_hr) {
 258   assert(!is_humongous(), "sanity / pre-condition");
 259   assert(top() == bottom(), "should be empty");
 260   assert(first_hr->is_starts_humongous(), "pre-condition");
 261 
 262   report_region_type_change(G1HeapRegionTraceType::ContinuesHumongous);
 263   _type.set_continues_humongous();
 264   _humongous_start_region = first_hr;
 265 
 266   _bot_part.set_object_can_span(true);
 267 }
 268 
 269 void HeapRegion::clear_humongous() {
 270   assert(is_humongous(), "pre-condition");
 271 
 272   assert(capacity() == HeapRegion::GrainBytes, "pre-condition");
 273   _humongous_start_region = NULL;
 274 
 275   _bot_part.set_object_can_span(false);
 276 }
 277 
 278 HeapRegion::HeapRegion(uint hrm_index,
 279                        G1BlockOffsetTable* bot,
 280                        MemRegion mr) :
 281     G1ContiguousSpace(bot),
 282     _hrm_index(hrm_index),
 283     _allocation_context(AllocationContext::system()),
 284     _humongous_start_region(NULL),
 285     _evacuation_failed(false),
 286     _prev_marked_bytes(0), _next_marked_bytes(0), _gc_efficiency(0.0),
 287     _next(NULL), _prev(NULL),
 288 #ifdef ASSERT
 289     _containing_set(NULL),
 290 #endif // ASSERT
 291      _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1),
 292     _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0)
 293 {
 294   _rem_set = new HeapRegionRemSet(bot, this);
 295 
 296   initialize(mr);
 297 }
 298 
 299 void HeapRegion::initialize(MemRegion mr, bool clear_space, bool mangle_space) {
 300   assert(_rem_set->is_empty(), "Remembered set must be empty");
 301 
 302   G1ContiguousSpace::initialize(mr, clear_space, mangle_space);
 303 
 304   hr_clear(false /*par*/, false /*clear_space*/);
 305   set_top(bottom());
 306   record_timestamp();
 307 }
 308 
 309 void HeapRegion::report_region_type_change(G1HeapRegionTraceType::Type to) {
 310   HeapRegionTracer::send_region_type_change(_hrm_index,
 311                                             get_trace_type(),
 312                                             to,
 313                                             (uintptr_t)bottom(),
 314                                             used(),
 315                                             (uint)allocation_context());
 316 }
 317 
 318 CompactibleSpace* HeapRegion::next_compaction_space() const {
 319   return G1CollectedHeap::heap()->next_compaction_region(this);
 320 }
 321 
 322 void HeapRegion::note_self_forwarding_removal_start(bool during_initial_mark,
 323                                                     bool during_conc_mark) {
 324   // We always recreate the prev marking info and we'll explicitly
 325   // mark all objects we find to be self-forwarded on the prev
 326   // bitmap. So all objects need to be below PTAMS.
 327   _prev_marked_bytes = 0;
 328 
 329   if (during_initial_mark) {
 330     // During initial-mark, we'll also explicitly mark all objects
 331     // we find to be self-forwarded on the next bitmap. So all
 332     // objects need to be below NTAMS.
 333     _next_top_at_mark_start = top();
 334     _next_marked_bytes = 0;
 335   } else if (during_conc_mark) {
 336     // During concurrent mark, all objects in the CSet (including
 337     // the ones we find to be self-forwarded) are implicitly live.
 338     // So all objects need to be above NTAMS.
 339     _next_top_at_mark_start = bottom();
 340     _next_marked_bytes = 0;
 341   }
 342 }
 343 
 344 void HeapRegion::note_self_forwarding_removal_end(size_t marked_bytes) {
 345   assert(marked_bytes <= used(),
 346          "marked: " SIZE_FORMAT " used: " SIZE_FORMAT, marked_bytes, used());
 347   _prev_top_at_mark_start = top();
 348   _prev_marked_bytes = marked_bytes;
 349 }
 350 
 351 // Humongous objects are allocated directly in the old-gen.  Need
 352 // special handling for concurrent processing encountering an
 353 // in-progress allocation.
 354 static bool do_oops_on_card_in_humongous(MemRegion mr,
 355                                          G1UpdateRSOrPushRefOopClosure* cl,
 356                                          HeapRegion* hr,
 357                                          G1CollectedHeap* g1h) {
 358   assert(hr->is_humongous(), "precondition");
 359   HeapRegion* sr = hr->humongous_start_region();
 360   oop obj = oop(sr->bottom());
 361 
 362   // If concurrent and klass_or_null is NULL, then space has been
 363   // allocated but the object has not yet been published by setting
 364   // the klass.  That can only happen if the card is stale.  However,
 365   // we've already set the card clean, so we must return failure,
 366   // since the allocating thread could have performed a write to the
 367   // card that might be missed otherwise.
 368   if (!g1h->is_gc_active() && (obj->klass_or_null_acquire() == NULL)) {
 369     return false;
 370   }
 371 
 372   // We have a well-formed humongous object at the start of sr.
 373   // Only filler objects follow a humongous object in the containing
 374   // regions, and we can ignore those.  So only process the one
 375   // humongous object.
 376   if (!g1h->is_obj_dead(obj, sr)) {
 377     if (obj->is_objArray() || (sr->bottom() < mr.start())) {
 378       // objArrays are always marked precisely, so limit processing
 379       // with mr.  Non-objArrays might be precisely marked, and since
 380       // it's humongous it's worthwhile avoiding full processing.
 381       // However, the card could be stale and only cover filler
 382       // objects.  That should be rare, so not worth checking for;
 383       // instead let it fall out from the bounded iteration.
 384       obj->oop_iterate(cl, mr);
 385     } else {
 386       // If obj is not an objArray and mr contains the start of the
 387       // obj, then this could be an imprecise mark, and we need to
 388       // process the entire object.
 389       obj->oop_iterate(cl);
 390     }
 391   }
 392   return true;
 393 }
 394 
 395 bool HeapRegion::oops_on_card_seq_iterate_careful(MemRegion mr,
 396                                                   G1UpdateRSOrPushRefOopClosure* cl) {
 397   assert(MemRegion(bottom(), end()).contains(mr), "Card region not in heap region");
 398   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 399 
 400   // Special handling for humongous regions.
 401   if (is_humongous()) {
 402     return do_oops_on_card_in_humongous(mr, cl, this, g1h);
 403   }
 404   assert(is_old(), "precondition");
 405 
 406   // Because mr has been trimmed to what's been allocated in this
 407   // region, the parts of the heap that are examined here are always
 408   // parsable; there's no need to use klass_or_null to detect
 409   // in-progress allocation.
 410 
 411   // Cache the boundaries of the memory region in some const locals
 412   HeapWord* const start = mr.start();
 413   HeapWord* const end = mr.end();
 414 
 415   // Find the obj that extends onto mr.start().
 416   // Update BOT as needed while finding start of (possibly dead)
 417   // object containing the start of the region.
 418   HeapWord* cur = block_start(start);
 419 
 420 #ifdef ASSERT
 421   {
 422     assert(cur <= start,
 423            "cur: " PTR_FORMAT ", start: " PTR_FORMAT, p2i(cur), p2i(start));
 424     HeapWord* next = cur + block_size(cur);
 425     assert(start < next,
 426            "start: " PTR_FORMAT ", next: " PTR_FORMAT, p2i(start), p2i(next));
 427   }
 428 #endif
 429 
 430   do {
 431     oop obj = oop(cur);
 432     assert(obj->is_oop(true), "Not an oop at " PTR_FORMAT, p2i(cur));
 433     assert(obj->klass_or_null() != NULL,
 434            "Unparsable heap at " PTR_FORMAT, p2i(cur));
 435 
 436     if (g1h->is_obj_dead(obj, this)) {
 437       // Carefully step over dead object.
 438       cur += block_size(cur);
 439     } else {
 440       // Step over live object, and process its references.
 441       cur += obj->size();
 442       // Non-objArrays are usually marked imprecise at the object
 443       // start, in which case we need to iterate over them in full.
 444       // objArrays are precisely marked, but can still be iterated
 445       // over in full if completely covered.
 446       if (!obj->is_objArray() || (((HeapWord*)obj) >= start && cur <= end)) {
 447         obj->oop_iterate(cl);
 448       } else {
 449         obj->oop_iterate(cl, mr);
 450       }
 451     }
 452   } while (cur < end);
 453 
 454   return true;
 455 }
 456 
 457 // Code roots support
 458 
 459 void HeapRegion::add_strong_code_root(nmethod* nm) {
 460   HeapRegionRemSet* hrrs = rem_set();
 461   hrrs->add_strong_code_root(nm);
 462 }
 463 
 464 void HeapRegion::add_strong_code_root_locked(nmethod* nm) {
 465   assert_locked_or_safepoint(CodeCache_lock);
 466   HeapRegionRemSet* hrrs = rem_set();
 467   hrrs->add_strong_code_root_locked(nm);
 468 }
 469 
 470 void HeapRegion::remove_strong_code_root(nmethod* nm) {
 471   HeapRegionRemSet* hrrs = rem_set();
 472   hrrs->remove_strong_code_root(nm);
 473 }
 474 
 475 void HeapRegion::strong_code_roots_do(CodeBlobClosure* blk) const {
 476   HeapRegionRemSet* hrrs = rem_set();
 477   hrrs->strong_code_roots_do(blk);
 478 }
 479 
 480 class VerifyStrongCodeRootOopClosure: public OopClosure {
 481   const HeapRegion* _hr;
 482   bool _failures;
 483   bool _has_oops_in_region;
 484 
 485   template <class T> void do_oop_work(T* p) {
 486     T heap_oop = oopDesc::load_heap_oop(p);
 487     if (!oopDesc::is_null(heap_oop)) {
 488       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
 489 
 490       // Note: not all the oops embedded in the nmethod are in the
 491       // current region. We only look at those which are.
 492       if (_hr->is_in(obj)) {
 493         // Object is in the region. Check that its less than top
 494         if (_hr->top() <= (HeapWord*)obj) {
 495           // Object is above top
 496           log_error(gc, verify)("Object " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ") is above top " PTR_FORMAT,
 497                                p2i(obj), p2i(_hr->bottom()), p2i(_hr->end()), p2i(_hr->top()));
 498           _failures = true;
 499           return;
 500         }
 501         // Nmethod has at least one oop in the current region
 502         _has_oops_in_region = true;
 503       }
 504     }
 505   }
 506 
 507 public:
 508   VerifyStrongCodeRootOopClosure(const HeapRegion* hr):
 509     _hr(hr), _failures(false), _has_oops_in_region(false) {}
 510 
 511   void do_oop(narrowOop* p) { do_oop_work(p); }
 512   void do_oop(oop* p)       { do_oop_work(p); }
 513 
 514   bool failures()           { return _failures; }
 515   bool has_oops_in_region() { return _has_oops_in_region; }
 516 };
 517 
 518 class VerifyStrongCodeRootCodeBlobClosure: public CodeBlobClosure {
 519   const HeapRegion* _hr;
 520   bool _failures;
 521 public:
 522   VerifyStrongCodeRootCodeBlobClosure(const HeapRegion* hr) :
 523     _hr(hr), _failures(false) {}
 524 
 525   void do_code_blob(CodeBlob* cb) {
 526     nmethod* nm = (cb == NULL) ? NULL : cb->as_compiled_method()->as_nmethod_or_null();
 527     if (nm != NULL) {
 528       // Verify that the nemthod is live
 529       if (!nm->is_alive()) {
 530         log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has dead nmethod " PTR_FORMAT " in its strong code roots",
 531                               p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
 532         _failures = true;
 533       } else {
 534         VerifyStrongCodeRootOopClosure oop_cl(_hr);
 535         nm->oops_do(&oop_cl);
 536         if (!oop_cl.has_oops_in_region()) {
 537           log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has nmethod " PTR_FORMAT " in its strong code roots with no pointers into region",
 538                                 p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
 539           _failures = true;
 540         } else if (oop_cl.failures()) {
 541           log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has other failures for nmethod " PTR_FORMAT,
 542                                 p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
 543           _failures = true;
 544         }
 545       }
 546     }
 547   }
 548 
 549   bool failures()       { return _failures; }
 550 };
 551 
 552 void HeapRegion::verify_strong_code_roots(VerifyOption vo, bool* failures) const {
 553   if (!G1VerifyHeapRegionCodeRoots) {
 554     // We're not verifying code roots.
 555     return;
 556   }
 557   if (vo == VerifyOption_G1UseMarkWord) {
 558     // Marking verification during a full GC is performed after class
 559     // unloading, code cache unloading, etc so the strong code roots
 560     // attached to each heap region are in an inconsistent state. They won't
 561     // be consistent until the strong code roots are rebuilt after the
 562     // actual GC. Skip verifying the strong code roots in this particular
 563     // time.
 564     assert(VerifyDuringGC, "only way to get here");
 565     return;
 566   }
 567 
 568   HeapRegionRemSet* hrrs = rem_set();
 569   size_t strong_code_roots_length = hrrs->strong_code_roots_list_length();
 570 
 571   // if this region is empty then there should be no entries
 572   // on its strong code root list
 573   if (is_empty()) {
 574     if (strong_code_roots_length > 0) {
 575       log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] is empty but has " SIZE_FORMAT " code root entries",
 576                             p2i(bottom()), p2i(end()), strong_code_roots_length);
 577       *failures = true;
 578     }
 579     return;
 580   }
 581 
 582   if (is_continues_humongous()) {
 583     if (strong_code_roots_length > 0) {
 584       log_error(gc, verify)("region " HR_FORMAT " is a continuation of a humongous region but has " SIZE_FORMAT " code root entries",
 585                             HR_FORMAT_PARAMS(this), strong_code_roots_length);
 586       *failures = true;
 587     }
 588     return;
 589   }
 590 
 591   VerifyStrongCodeRootCodeBlobClosure cb_cl(this);
 592   strong_code_roots_do(&cb_cl);
 593 
 594   if (cb_cl.failures()) {
 595     *failures = true;
 596   }
 597 }
 598 
 599 void HeapRegion::print() const { print_on(tty); }
 600 void HeapRegion::print_on(outputStream* st) const {
 601   st->print("|%4u", this->_hrm_index);
 602   st->print("|" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT,
 603             p2i(bottom()), p2i(top()), p2i(end()));
 604   st->print("|%3d%%", (int) ((double) used() * 100 / capacity()));
 605   st->print("|%2s", get_short_type_str());
 606   if (in_collection_set()) {
 607     st->print("|CS");
 608   } else {
 609     st->print("|  ");
 610   }
 611   st->print("|TS%3u", _gc_time_stamp);
 612   st->print("|AC%3u", allocation_context());
 613   st->print_cr("|TAMS " PTR_FORMAT ", " PTR_FORMAT "|",
 614                p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start()));
 615 }
 616 
 617 class G1VerificationClosure : public OopClosure {
 618 protected:
 619   G1CollectedHeap* _g1h;
 620   G1CardTable *_ct;
 621   oop _containing_obj;
 622   bool _failures;
 623   int _n_failures;
 624   VerifyOption _vo;
 625 public:
 626   // _vo == UsePrevMarking -> use "prev" marking information,
 627   // _vo == UseNextMarking -> use "next" marking information,
 628   // _vo == UseMarkWord    -> use mark word from object header.
 629   G1VerificationClosure(G1CollectedHeap* g1h, VerifyOption vo) :
 630     _g1h(g1h), _ct(g1h->g1_card_table()),
 631     _containing_obj(NULL), _failures(false), _n_failures(0), _vo(vo) {
 632   }
 633 
 634   void set_containing_obj(oop obj) {
 635     _containing_obj = obj;
 636   }
 637 
 638   bool failures() { return _failures; }
 639   int n_failures() { return _n_failures; }
 640 
 641   void print_object(outputStream* out, oop obj) {
 642 #ifdef PRODUCT
 643     Klass* k = obj->klass();
 644     const char* class_name = k->external_name();
 645     out->print_cr("class name %s", class_name);
 646 #else // PRODUCT
 647     obj->print_on(out);
 648 #endif // PRODUCT
 649   }
 650 };
 651 
 652 class VerifyLiveClosure : public G1VerificationClosure {
 653 public:
 654   VerifyLiveClosure(G1CollectedHeap* g1h, VerifyOption vo) : G1VerificationClosure(g1h, vo) {}
 655   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
 656   virtual void do_oop(oop* p) { do_oop_work(p); }
 657 
 658   template <class T>
 659   void do_oop_work(T* p) {
 660     assert(_containing_obj != NULL, "Precondition");
 661     assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo),
 662       "Precondition");
 663     verify_liveness(p);
 664   }
 665 
 666   template <class T>
 667   void verify_liveness(T* p) {
 668     T heap_oop = oopDesc::load_heap_oop(p);
 669     Log(gc, verify) log;
 670     if (!oopDesc::is_null(heap_oop)) {
 671       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
 672       bool failed = false;
 673       if (!_g1h->is_in_closed_subset(obj) || _g1h->is_obj_dead_cond(obj, _vo)) {
 674         MutexLockerEx x(ParGCRareEvent_lock,
 675           Mutex::_no_safepoint_check_flag);
 676 
 677         if (!_failures) {
 678           log.error("----------");
 679         }
 680         ResourceMark rm;
 681         if (!_g1h->is_in_closed_subset(obj)) {
 682           HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
 683           log.error("Field " PTR_FORMAT " of live obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
 684             p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
 685           print_object(log.error_stream(), _containing_obj);
 686           log.error("points to obj " PTR_FORMAT " not in the heap", p2i(obj));
 687         } else {
 688           HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
 689           HeapRegion* to = _g1h->heap_region_containing((HeapWord*)obj);
 690           log.error("Field " PTR_FORMAT " of live obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
 691             p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
 692           print_object(log.error_stream(), _containing_obj);
 693           log.error("points to dead obj " PTR_FORMAT " in region [" PTR_FORMAT ", " PTR_FORMAT ")",
 694             p2i(obj), p2i(to->bottom()), p2i(to->end()));
 695           print_object(log.error_stream(), obj);
 696         }
 697         log.error("----------");
 698         _failures = true;
 699         failed = true;
 700         _n_failures++;
 701       }
 702     }
 703   }
 704 };
 705 
 706 class VerifyRemSetClosure : public G1VerificationClosure {
 707 public:
 708   VerifyRemSetClosure(G1CollectedHeap* g1h, VerifyOption vo) : G1VerificationClosure(g1h, vo) {}
 709   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
 710   virtual void do_oop(oop* p) { do_oop_work(p); }
 711 
 712   template <class T>
 713   void do_oop_work(T* p) {
 714     assert(_containing_obj != NULL, "Precondition");
 715     assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo),
 716       "Precondition");
 717     verify_remembered_set(p);
 718   }
 719 
 720   template <class T>
 721   void verify_remembered_set(T* p) {
 722     T heap_oop = oopDesc::load_heap_oop(p);
 723     Log(gc, verify) log;
 724     if (!oopDesc::is_null(heap_oop)) {
 725       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
 726       HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
 727       HeapRegion* to = _g1h->heap_region_containing(obj);
 728       if (from != NULL && to != NULL &&
 729         from != to &&
 730         !to->is_pinned()) {
 731         jbyte cv_obj = *_ct->byte_for_const(_containing_obj);
 732         jbyte cv_field = *_ct->byte_for_const(p);
 733         const jbyte dirty = G1CardTable::dirty_card_val();
 734 
 735         bool is_bad = !(from->is_young()
 736           || to->rem_set()->contains_reference(p)
 737           || !G1HRRSFlushLogBuffersOnVerify && // buffers were not flushed
 738           (_containing_obj->is_objArray() ?
 739           cv_field == dirty
 740           : cv_obj == dirty || cv_field == dirty));
 741         if (is_bad) {
 742           MutexLockerEx x(ParGCRareEvent_lock,
 743             Mutex::_no_safepoint_check_flag);
 744 
 745           if (!_failures) {
 746             log.error("----------");
 747           }
 748           log.error("Missing rem set entry:");
 749           log.error("Field " PTR_FORMAT " of obj " PTR_FORMAT ", in region " HR_FORMAT,
 750             p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
 751           ResourceMark rm;
 752           _containing_obj->print_on(log.error_stream());
 753           log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT, p2i(obj), HR_FORMAT_PARAMS(to));
 754           if (obj->is_oop()) {
 755             obj->print_on(log.error_stream());
 756           }
 757           log.error("Obj head CTE = %d, field CTE = %d.", cv_obj, cv_field);
 758           log.error("----------");
 759           _failures = true;
 760           _n_failures++;
 761         }
 762       }
 763     }
 764   }
 765 };
 766 
 767 // Closure that applies the given two closures in sequence.
 768 class G1Mux2Closure : public OopClosure {
 769   OopClosure* _c1;
 770   OopClosure* _c2;
 771 public:
 772   G1Mux2Closure(OopClosure *c1, OopClosure *c2) { _c1 = c1; _c2 = c2; }
 773   template <class T> inline void do_oop_work(T* p) {
 774     // Apply first closure; then apply the second.
 775     _c1->do_oop(p);
 776     _c2->do_oop(p);
 777   }
 778   virtual inline void do_oop(oop* p) { do_oop_work(p); }
 779   virtual inline void do_oop(narrowOop* p) { do_oop_work(p); }
 780 };
 781 
 782 // This really ought to be commoned up into OffsetTableContigSpace somehow.
 783 // We would need a mechanism to make that code skip dead objects.
 784 
 785 void HeapRegion::verify(VerifyOption vo,
 786                         bool* failures) const {
 787   G1CollectedHeap* g1 = G1CollectedHeap::heap();
 788   *failures = false;
 789   HeapWord* p = bottom();
 790   HeapWord* prev_p = NULL;
 791   VerifyLiveClosure vl_cl(g1, vo);
 792   VerifyRemSetClosure vr_cl(g1, vo);
 793   bool is_region_humongous = is_humongous();
 794   size_t object_num = 0;
 795   while (p < top()) {
 796     oop obj = oop(p);
 797     size_t obj_size = block_size(p);
 798     object_num += 1;
 799 
 800     if (!g1->is_obj_dead_cond(obj, this, vo)) {
 801       if (obj->is_oop()) {
 802         Klass* klass = obj->klass();
 803         bool is_metaspace_object = Metaspace::contains(klass) ||
 804                                    (vo == VerifyOption_G1UsePrevMarking &&
 805                                    ClassLoaderDataGraph::unload_list_contains(klass));
 806         if (!is_metaspace_object) {
 807           log_error(gc, verify)("klass " PTR_FORMAT " of object " PTR_FORMAT " "
 808                                 "not metadata", p2i(klass), p2i(obj));
 809           *failures = true;
 810           return;
 811         } else if (!klass->is_klass()) {
 812           log_error(gc, verify)("klass " PTR_FORMAT " of object " PTR_FORMAT " "
 813                                 "not a klass", p2i(klass), p2i(obj));
 814           *failures = true;
 815           return;
 816         } else {
 817           vl_cl.set_containing_obj(obj);
 818           if (!g1->collector_state()->full_collection() || G1VerifyRSetsDuringFullGC) {
 819             // verify liveness and rem_set
 820             vr_cl.set_containing_obj(obj);
 821             G1Mux2Closure mux(&vl_cl, &vr_cl);
 822             obj->oop_iterate_no_header(&mux);
 823 
 824             if (vr_cl.failures()) {
 825               *failures = true;
 826             }
 827             if (G1MaxVerifyFailures >= 0 &&
 828               vr_cl.n_failures() >= G1MaxVerifyFailures) {
 829               return;
 830             }
 831           } else {
 832             // verify only liveness
 833             obj->oop_iterate_no_header(&vl_cl);
 834           }
 835           if (vl_cl.failures()) {
 836             *failures = true;
 837           }
 838           if (G1MaxVerifyFailures >= 0 &&
 839               vl_cl.n_failures() >= G1MaxVerifyFailures) {
 840             return;
 841           }
 842         }
 843       } else {
 844         log_error(gc, verify)(PTR_FORMAT " not an oop", p2i(obj));
 845         *failures = true;
 846         return;
 847       }
 848     }
 849     prev_p = p;
 850     p += obj_size;
 851   }
 852 
 853   if (!is_young() && !is_empty()) {
 854     _bot_part.verify();
 855   }
 856 
 857   if (is_region_humongous) {
 858     oop obj = oop(this->humongous_start_region()->bottom());
 859     if ((HeapWord*)obj > bottom() || (HeapWord*)obj + obj->size() < bottom()) {
 860       log_error(gc, verify)("this humongous region is not part of its' humongous object " PTR_FORMAT, p2i(obj));
 861       *failures = true;
 862       return;
 863     }
 864   }
 865 
 866   if (!is_region_humongous && p != top()) {
 867     log_error(gc, verify)("end of last object " PTR_FORMAT " "
 868                           "does not match top " PTR_FORMAT, p2i(p), p2i(top()));
 869     *failures = true;
 870     return;
 871   }
 872 
 873   HeapWord* the_end = end();
 874   // Do some extra BOT consistency checking for addresses in the
 875   // range [top, end). BOT look-ups in this range should yield
 876   // top. No point in doing that if top == end (there's nothing there).
 877   if (p < the_end) {
 878     // Look up top
 879     HeapWord* addr_1 = p;
 880     HeapWord* b_start_1 = _bot_part.block_start_const(addr_1);
 881     if (b_start_1 != p) {
 882       log_error(gc, verify)("BOT look up for top: " PTR_FORMAT " "
 883                             " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
 884                             p2i(addr_1), p2i(b_start_1), p2i(p));
 885       *failures = true;
 886       return;
 887     }
 888 
 889     // Look up top + 1
 890     HeapWord* addr_2 = p + 1;
 891     if (addr_2 < the_end) {
 892       HeapWord* b_start_2 = _bot_part.block_start_const(addr_2);
 893       if (b_start_2 != p) {
 894         log_error(gc, verify)("BOT look up for top + 1: " PTR_FORMAT " "
 895                               " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
 896                               p2i(addr_2), p2i(b_start_2), p2i(p));
 897         *failures = true;
 898         return;
 899       }
 900     }
 901 
 902     // Look up an address between top and end
 903     size_t diff = pointer_delta(the_end, p) / 2;
 904     HeapWord* addr_3 = p + diff;
 905     if (addr_3 < the_end) {
 906       HeapWord* b_start_3 = _bot_part.block_start_const(addr_3);
 907       if (b_start_3 != p) {
 908         log_error(gc, verify)("BOT look up for top + diff: " PTR_FORMAT " "
 909                               " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
 910                               p2i(addr_3), p2i(b_start_3), p2i(p));
 911         *failures = true;
 912         return;
 913       }
 914     }
 915 
 916     // Look up end - 1
 917     HeapWord* addr_4 = the_end - 1;
 918     HeapWord* b_start_4 = _bot_part.block_start_const(addr_4);
 919     if (b_start_4 != p) {
 920       log_error(gc, verify)("BOT look up for end - 1: " PTR_FORMAT " "
 921                             " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
 922                             p2i(addr_4), p2i(b_start_4), p2i(p));
 923       *failures = true;
 924       return;
 925     }
 926   }
 927 
 928   verify_strong_code_roots(vo, failures);
 929 }
 930 
 931 void HeapRegion::verify() const {
 932   bool dummy = false;
 933   verify(VerifyOption_G1UsePrevMarking, /* failures */ &dummy);
 934 }
 935 
 936 void HeapRegion::verify_rem_set(VerifyOption vo, bool* failures) const {
 937   G1CollectedHeap* g1 = G1CollectedHeap::heap();
 938   *failures = false;
 939   HeapWord* p = bottom();
 940   HeapWord* prev_p = NULL;
 941   VerifyRemSetClosure vr_cl(g1, vo);
 942   while (p < top()) {
 943     oop obj = oop(p);
 944     size_t obj_size = block_size(p);
 945 
 946     if (!g1->is_obj_dead_cond(obj, this, vo)) {
 947       if (obj->is_oop()) {
 948         vr_cl.set_containing_obj(obj);
 949         obj->oop_iterate_no_header(&vr_cl);
 950 
 951         if (vr_cl.failures()) {
 952           *failures = true;
 953         }
 954         if (G1MaxVerifyFailures >= 0 &&
 955           vr_cl.n_failures() >= G1MaxVerifyFailures) {
 956           return;
 957         }
 958       } else {
 959         log_error(gc, verify)(PTR_FORMAT " not an oop", p2i(obj));
 960         *failures = true;
 961         return;
 962       }
 963     }
 964 
 965     prev_p = p;
 966     p += obj_size;
 967   }
 968 }
 969 
 970 void HeapRegion::verify_rem_set() const {
 971   bool failures = false;
 972   verify_rem_set(VerifyOption_G1UsePrevMarking, &failures);
 973   guarantee(!failures, "HeapRegion RemSet verification failed");
 974 }
 975 
 976 void HeapRegion::prepare_for_compaction(CompactPoint* cp) {
 977   scan_and_forward(this, cp);
 978 }
 979 
 980 // G1OffsetTableContigSpace code; copied from space.cpp.  Hope this can go
 981 // away eventually.
 982 
 983 void G1ContiguousSpace::clear(bool mangle_space) {
 984   set_top(bottom());
 985   _scan_top = bottom();
 986   CompactibleSpace::clear(mangle_space);
 987   reset_bot();
 988 }
 989 
 990 #ifndef PRODUCT
 991 void G1ContiguousSpace::mangle_unused_area() {
 992   mangle_unused_area_complete();
 993 }
 994 
 995 void G1ContiguousSpace::mangle_unused_area_complete() {
 996   SpaceMangler::mangle_region(MemRegion(top(), end()));
 997 }
 998 #endif
 999 
1000 void G1ContiguousSpace::print() const {
1001   print_short();
1002   tty->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", "
1003                 INTPTR_FORMAT ", " INTPTR_FORMAT ")",
1004                 p2i(bottom()), p2i(top()), p2i(_bot_part.threshold()), p2i(end()));
1005 }
1006 
1007 HeapWord* G1ContiguousSpace::initialize_threshold() {
1008   return _bot_part.initialize_threshold();
1009 }
1010 
1011 HeapWord* G1ContiguousSpace::cross_threshold(HeapWord* start,
1012                                                     HeapWord* end) {
1013   _bot_part.alloc_block(start, end);
1014   return _bot_part.threshold();
1015 }
1016 
1017 HeapWord* G1ContiguousSpace::scan_top() const {
1018   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1019   HeapWord* local_top = top();
1020   OrderAccess::loadload();
1021   const unsigned local_time_stamp = _gc_time_stamp;
1022   assert(local_time_stamp <= g1h->get_gc_time_stamp(), "invariant");
1023   if (local_time_stamp < g1h->get_gc_time_stamp()) {
1024     return local_top;
1025   } else {
1026     return _scan_top;
1027   }
1028 }
1029 
1030 void G1ContiguousSpace::record_timestamp() {
1031   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1032   uint curr_gc_time_stamp = g1h->get_gc_time_stamp();
1033 
1034   if (_gc_time_stamp < curr_gc_time_stamp) {
1035     // Setting the time stamp here tells concurrent readers to look at
1036     // scan_top to know the maximum allowed address to look at.
1037 
1038     // scan_top should be bottom for all regions except for the
1039     // retained old alloc region which should have scan_top == top
1040     HeapWord* st = _scan_top;
1041     guarantee(st == _bottom || st == _top, "invariant");
1042 
1043     _gc_time_stamp = curr_gc_time_stamp;
1044   }
1045 }
1046 
1047 void G1ContiguousSpace::record_retained_region() {
1048   // scan_top is the maximum address where it's safe for the next gc to
1049   // scan this region.
1050   _scan_top = top();
1051 }
1052 
1053 void G1ContiguousSpace::safe_object_iterate(ObjectClosure* blk) {
1054   object_iterate(blk);
1055 }
1056 
1057 void G1ContiguousSpace::object_iterate(ObjectClosure* blk) {
1058   HeapWord* p = bottom();
1059   while (p < top()) {
1060     if (block_is_obj(p)) {
1061       blk->do_object(oop(p));
1062     }
1063     p += block_size(p);
1064   }
1065 }
1066 
1067 G1ContiguousSpace::G1ContiguousSpace(G1BlockOffsetTable* bot) :
1068   _bot_part(bot, this),
1069   _par_alloc_lock(Mutex::leaf, "OffsetTableContigSpace par alloc lock", true),
1070   _gc_time_stamp(0)
1071 {
1072 }
1073 
1074 void G1ContiguousSpace::initialize(MemRegion mr, bool clear_space, bool mangle_space) {
1075   CompactibleSpace::initialize(mr, clear_space, mangle_space);
1076   _top = bottom();
1077   _scan_top = bottom();
1078   set_saved_mark_word(NULL);
1079   reset_bot();
1080 }