src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp

rev 10658 : [backport] Single marking bitmap
rev 10659 : [backport] Make sure bitmap is marked incomplete before bitmap resets
rev 10660 : [backport] Refactor bitmap cleaning
rev 10664 : [backport] Fix indenting in ShMarkCompact::do_it
rev 10665 : [backport] Coarsen Full GC use of heap lock
rev 10668 : [backport] Cleanup buffered queue handling
rev 10678 : [backport] Rename concurrentMark -> concurrent_mark
rev 10690 : [backport] Cleanup header files and forward declarations
rev 10695 : [backport] Inline/rename confusingly named methods in ShConcurrentMark
rev 10715 : [backport] Cleanup up superfluous newlines
rev 10737 : [backport] Support VerifyBeforeGC and VerifyAfterGC VM options
rev 10743 : [backport] Clean up ShHeap::heap_region_iterate uses
rev 10749 : [backport] Adjust metaspace size during each GC
rev 10755 : [backport] Make heuristics tell if we can process references or unload classes
rev 10764 : [backport] Rename BrooksPointer to ShenandoahBrooksPointer
rev 10772 : [backport] Update copyrights
rev 10801 : [backport] Rename vm_operations_shenandoah.* to shenandoahVMOperations.*

@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2014, 2015, Red Hat, Inc. and/or its affiliates.
+ * Copyright (c) 2014, 2018, Red Hat, Inc. All rights reserved.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
  * published by the Free Software Foundation.
  *

@@ -20,50 +20,56 @@
  * questions.
  *
  */
 
 #include "precompiled.hpp"
+
 #include "code/codeCache.hpp"
 #include "gc_implementation/shenandoah/shenandoahGCTraceTime.hpp"
 #include "gc_implementation/shared/gcTimer.hpp"
-#include "gc_implementation/shenandoah/brooksPointer.hpp"
+#include "gc_implementation/shenandoah/shenandoahBrooksPointer.hpp"
 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp"
 #include "gc_implementation/shenandoah/shenandoahFreeSet.hpp"
 #include "gc_implementation/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc_implementation/shenandoah/shenandoahMarkCompact.hpp"
-#include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeap.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc_implementation/shenandoah/shenandoahHeuristics.hpp"
 #include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp"
+#include "gc_implementation/shenandoah/shenandoahTaskqueue.hpp"
+#include "gc_implementation/shenandoah/shenandoahTaskqueue.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahUtils.hpp"
 #include "gc_implementation/shenandoah/shenandoahVerifier.hpp"
+#include "gc_implementation/shenandoah/shenandoahVMOperations.hpp"
+#include "gc_implementation/shenandoah/shenandoahWorkGroup.hpp"
 #include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp"
-#include "gc_implementation/shenandoah/vm_operations_shenandoah.hpp"
+#include "memory/metaspace.hpp"
 #include "oops/oop.inline.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/thread.hpp"
 #include "utilities/copy.hpp"
 #include "utilities/growableArray.hpp"
-#include "utilities/taskqueue.hpp"
 #include "utilities/workgroup.hpp"
 
 void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
   _gc_timer = gc_timer;
 }
 
 void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 
-  {
     if (ShenandoahVerify) {
       heap->verifier()->verify_before_fullgc();
     }
 
+  if (VerifyBeforeGC) {
+    Universe::verify();
+  }
+
     heap->set_full_gc_in_progress(true);
 
     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
     assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
 

@@ -79,11 +85,11 @@
       // 0. Remember if we have forwarded objects
       bool has_forwarded_objects = heap->has_forwarded_objects();
 
       // a. Cancel concurrent mark, if in progress
       if (heap->is_concurrent_mark_in_progress()) {
-        heap->concurrentMark()->cancel();
+      heap->concurrent_mark()->cancel();
         heap->stop_concurrent_marking();
       }
       assert(!heap->is_concurrent_mark_in_progress(), "sanity");
 
       // b1. Cancel evacuation, if in progress

@@ -97,12 +103,13 @@
         heap->set_update_refs_in_progress(false);
       }
       assert(!heap->is_update_refs_in_progress(), "sanity");
 
       // c. Reset the bitmaps for new marking
-      heap->reset_next_mark_bitmap();
-      assert(heap->next_marking_context()->is_bitmap_clear(), "sanity");
+    heap->reset_mark_bitmap();
+    assert(heap->marking_context()->is_bitmap_clear(), "sanity");
+    assert(!heap->marking_context()->is_complete(), "sanity");
 
       // d. Abandon reference discovery and clear all discovered references.
       ReferenceProcessor *rp = heap->ref_processor();
       rp->disable_discovery();
       rp->abandon_partial_discovery();

@@ -110,11 +117,10 @@
 
       // e. Set back forwarded objects bit back, in case some steps above dropped it.
       heap->set_has_forwarded_objects(has_forwarded_objects);
     }
 
-    {
       heap->make_parsable(true);
 
       CodeCache::gc_prologue();
 
       OrderAccess::fence();

@@ -127,43 +133,53 @@
       heap->set_has_forwarded_objects(false);
 
       heap->set_full_gc_move_in_progress(true);
 
       // Setup workers for the rest
-      {
         OrderAccess::fence();
 
         // Initialize worker slices
         ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
         for (uint i = 0; i < heap->max_workers(); i++) {
           worker_slices[i] = new ShenandoahHeapRegionSet();
         }
 
+  {
+    // The rest of code performs region moves, where region status is undefined
+    // until all phases run together.
+    ShenandoahHeapLocker lock(heap->lock());
+
         phase2_calculate_target_addresses(worker_slices);
 
         OrderAccess::fence();
 
         phase3_update_references();
 
         phase4_compact_objects(worker_slices);
+  }
+
+  // Resize metaspace
+  MetaspaceGC::compute_new_size();
 
         // Free worker slices
         for (uint i = 0; i < heap->max_workers(); i++) {
           delete worker_slices[i];
         }
         FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices, mtGC);
 
         CodeCache::gc_epilogue();
         JvmtiExport::gc_epilogue();
-      }
 
       heap->set_full_gc_move_in_progress(false);
       heap->set_full_gc_in_progress(false);
 
       if (ShenandoahVerify) {
         heap->verifier()->verify_after_fullgc();
       }
+
+  if (VerifyAfterGC) {
+    Universe::verify();
     }
 
     {
       ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
       heap->post_full_gc_dump(_gc_timer);

@@ -171,57 +187,50 @@
 
     if (UseTLAB) {
       ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
       heap->resize_all_tlabs();
     }
-  }
 }
 
 class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
 private:
   ShenandoahMarkingContext* const _ctx;
 
 public:
-  ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->next_marking_context()) {}
+  ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
 
-  bool heap_region_do(ShenandoahHeapRegion *r) {
-    _ctx->set_top_at_mark_start(r->region_number(), r->top());
+  void heap_region_do(ShenandoahHeapRegion *r) {
+    _ctx->capture_top_at_mark_start(r);
     r->clear_live_data();
     r->set_concurrent_iteration_safe_limit(r->top());
-    return false;
   }
 };
 
 void ShenandoahMarkCompact::phase1_mark_heap() {
   ShenandoahHeap* heap = ShenandoahHeap::heap();
   GCTraceTime time("Phase 1: Mark live objects", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
   ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
 
-  {
-    ShenandoahHeapLocker lock(heap->lock());
     ShenandoahPrepareForMarkClosure cl;
-    heap->heap_region_iterate(&cl, false, false);
-  }
+  heap->heap_region_iterate(&cl);
 
-  ShenandoahConcurrentMark* cm = heap->concurrentMark();
+  ShenandoahConcurrentMark* cm = heap->concurrent_mark();
 
-  // Do not trust heuristics, because this can be our last resort collection.
-  // Only ignore processing references and class unloading if explicitly disabled.
-  heap->set_process_references(ShenandoahRefProcFrequency != 0);
-  heap->set_unload_classes(ClassUnloading);
+  heap->set_process_references(heap->heuristics()->can_process_references());
+  heap->set_unload_classes(heap->heuristics()->can_unload_classes());
 
   ReferenceProcessor* rp = heap->ref_processor();
   // enable ("weak") refs discovery
   rp->enable_discovery(true /*verify_no_refs*/, true);
   rp->setup_policy(true); // forcefully purge all soft references
   rp->set_active_mt_degree(heap->workers()->active_workers());
 
   cm->update_roots(ShenandoahPhaseTimings::full_gc_roots);
   cm->mark_roots(ShenandoahPhaseTimings::full_gc_roots);
-  cm->shared_finish_mark_from_roots(/* full_gc = */ true);
+  cm->finish_mark_from_roots(/* full_gc = */ true);
 
-  heap->swap_mark_contexts();
+  heap->mark_complete_marking_context();
 }
 
 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
 private:
   ShenandoahHeap*          const _heap;

@@ -260,11 +269,11 @@
   void do_object(oop p) {
     assert(_from_region != NULL, "must set before work");
     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
     assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
 
-    size_t obj_size = p->size() + BrooksPointer::word_size();
+    size_t obj_size = p->size() + ShenandoahBrooksPointer::word_size();
     if (_compact_point + obj_size > _to_region->end()) {
       finish_region();
 
       // Object doesn't fit. Pick next empty region and start compacting there.
       ShenandoahHeapRegion* new_to_region;

@@ -283,11 +292,11 @@
     }
 
     // Object fits into current region, record new location:
     assert(_compact_point + obj_size <= _to_region->end(), "must fit");
     shenandoah_assert_not_forwarded(NULL, p);
-    BrooksPointer::set_raw(p, _compact_point + BrooksPointer::word_size());
+    ShenandoahBrooksPointer::set_raw(p, _compact_point + ShenandoahBrooksPointer::word_size());
     _compact_point += obj_size;
   }
 };
 
 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {

@@ -380,19 +389,19 @@
       continue;
     }
 
     if (r->is_humongous_start() && r->is_move_allowed()) {
       // From-region candidate: movable humongous region
-      oop old_obj = oop(r->bottom() + BrooksPointer::word_size());
-      size_t words_size = old_obj->size() + BrooksPointer::word_size();
+      oop old_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
+      size_t words_size = old_obj->size() + ShenandoahBrooksPointer::word_size();
       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 
       size_t start = to_end - num_regions;
 
       if (start >= to_begin && start != r->region_number()) {
         // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
-        BrooksPointer::set_raw(old_obj, heap->get_region(start)->bottom() + BrooksPointer::word_size());
+        ShenandoahBrooksPointer::set_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahBrooksPointer::word_size());
         to_end = start;
         continue;
       }
     }
 

@@ -406,11 +415,11 @@
 private:
   ShenandoahHeap* const _heap;
 
 public:
   ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
-  bool heap_region_do(ShenandoahHeapRegion* r) {
+  void heap_region_do(ShenandoahHeapRegion* r) {
     if (r->is_trash()) {
       r->recycle();
     }
     if (r->is_cset()) {
       r->make_regular_bypass();

@@ -421,11 +430,10 @@
     assert (r->is_committed(), err_msg("only committed regions in heap now, see region " SIZE_FORMAT, r->region_number()));
 
     // Record current region occupancy: this communicates empty regions are free
     // to the rest of Full GC code.
     r->set_new_top(r->top());
-    return false;
   }
 };
 
 class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
 private:

@@ -435,13 +443,13 @@
 public:
   ShenandoahTrashImmediateGarbageClosure() :
     _heap(ShenandoahHeap::heap()),
     _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
 
-  bool heap_region_do(ShenandoahHeapRegion* r) {
+  void heap_region_do(ShenandoahHeapRegion* r) {
     if (r->is_humongous_start()) {
-      oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
+      oop humongous_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
       if (!_ctx->is_marked(humongous_obj)) {
         assert(!r->has_live(),
                err_msg("Region " SIZE_FORMAT " is not marked, should not have live", r->region_number()));
         _heap->trash_humongous_region_at(r);
       } else {

@@ -452,35 +460,30 @@
       // If we hit continuation, the non-live humongous starts should have been trashed already
       assert(r->humongous_start_region()->has_live(),
              err_msg("Region " SIZE_FORMAT " should have live", r->region_number()));
     } else if (r->is_regular()) {
       if (!r->has_live()) {
-        assert(_ctx->is_bitmap_clear_range(r->bottom(), r->end()),
-               err_msg("Region " SIZE_FORMAT " should not have marks in bitmap", r->region_number()));
-        r->make_trash();
+        r->make_trash_immediate();
       }
     }
-    return false;
   }
 };
 
 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
   ShenandoahHeap* heap = ShenandoahHeap::heap();
   GCTraceTime time("Phase 2: Compute new object addresses", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
   ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
 
   {
-    ShenandoahHeapLocker lock(heap->lock());
-
     // Trash the immediately collectible regions before computing addresses
     ShenandoahTrashImmediateGarbageClosure tigcl;
-    heap->heap_region_iterate(&tigcl, false, false);
+    heap->heap_region_iterate(&tigcl);
 
     // Make sure regions are in good state: committed, active, clean.
     // This is needed because we are potentially sliding the data through them.
     ShenandoahEnsureHeapActiveClosure ecl;
-    heap->heap_region_iterate(&ecl, false, false);
+    heap->heap_region_iterate(&ecl);
   }
 
   // Compute the new addresses for regular objects
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);

@@ -504,11 +507,11 @@
   inline void do_oop_work(T* p) {
     T o = oopDesc::load_heap_oop(p);
     if (! oopDesc::is_null(o)) {
       oop obj = oopDesc::decode_heap_oop_not_null(o);
       assert(_ctx->is_marked(obj), "must be marked");
-      oop forw = oop(BrooksPointer::get_raw(obj));
+      oop forw = oop(ShenandoahBrooksPointer::get_raw(obj));
       oopDesc::encode_store_heap_oop(p, forw);
     }
   }
 
 public:

@@ -606,17 +609,17 @@
   ShenandoahCompactObjectsClosure() : _heap(ShenandoahHeap::heap()) {}
 
   void do_object(oop p) {
     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
     size_t size = (size_t)p->size();
-    HeapWord* compact_to = BrooksPointer::get_raw(p);
+    HeapWord* compact_to = ShenandoahBrooksPointer::get_raw(p);
     HeapWord* compact_from = (HeapWord*) p;
     if (compact_from != compact_to) {
       Copy::aligned_conjoint_words(compact_from, compact_to, size);
     }
     oop new_obj = oop(compact_to);
-    BrooksPointer::initialize(new_obj);
+    ShenandoahBrooksPointer::initialize(new_obj);
   }
 };
 
 class ShenandoahCompactObjectsTask : public AbstractGangTask {
 private:

@@ -654,20 +657,20 @@
 public:
   ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
     _heap->free_set()->clear();
   }
 
-  bool heap_region_do(ShenandoahHeapRegion* r) {
+  void heap_region_do(ShenandoahHeapRegion* r) {
     assert (!r->is_cset(), "cset regions should have been demoted already");
 
     // Need to reset the complete-top-at-mark-start pointer here because
     // the complete marking bitmap is no longer valid. This ensures
     // size-based iteration in marked_object_iterate().
     // NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
     // pinned regions.
     if (!r->is_pinned()) {
-      _heap->complete_marking_context()->set_top_at_mark_start(r->region_number(), r->bottom());
+      _heap->complete_marking_context()->reset_top_at_mark_start(r);
     }
 
     size_t live = r->used();
 
     // Make empty regions that have been allocated into regular

@@ -687,11 +690,10 @@
     }
 
     r->set_live_data(live);
     r->reset_alloc_metadata_to_shared();
     _live += live;
-    return false;
   }
 
   size_t get_live() {
     return _live;
   }

@@ -707,17 +709,17 @@
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 
   for (size_t c = heap->num_regions() - 1; c > 0; c--) {
     ShenandoahHeapRegion* r = heap->get_region(c);
     if (r->is_humongous_start()) {
-      oop old_obj = oop(r->bottom() + BrooksPointer::word_size());
-      size_t words_size = old_obj->size() + BrooksPointer::word_size();
+      oop old_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
+      size_t words_size = old_obj->size() + ShenandoahBrooksPointer::word_size();
       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 
       size_t old_start = r->region_number();
       size_t old_end   = old_start + num_regions - 1;
-      size_t new_start = heap->heap_region_index_containing(BrooksPointer::get_raw(old_obj));
+      size_t new_start = heap->heap_region_index_containing(ShenandoahBrooksPointer::get_raw(old_obj));
       size_t new_end   = new_start + num_regions - 1;
 
       if (old_start == new_start) {
         // No need to move the object, it stays at the same slot
         continue;

@@ -727,16 +729,14 @@
 
       Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
                                    heap->get_region(new_start)->bottom(),
                                    ShenandoahHeapRegion::region_size_words()*num_regions);
 
-      oop new_obj = oop(heap->get_region(new_start)->bottom() + BrooksPointer::word_size());
-      BrooksPointer::initialize(new_obj);
+      oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahBrooksPointer::word_size());
+      ShenandoahBrooksPointer::initialize(new_obj);
 
       {
-        ShenandoahHeapLocker lock(heap->lock());
-
         for (size_t c = old_start; c <= old_end; c++) {
           ShenandoahHeapRegion* r = heap->get_region(c);
           r->make_regular_bypass();
           r->set_top(r->bottom());
         }

@@ -762,11 +762,10 @@
       }
     }
   }
 }
 
-
 // This is slightly different to ShHeap::reset_next_mark_bitmap:
 // we need to remain able to walk pinned regions.
 // Since pinned region do not move and don't get compacted, we will get holes with
 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
 // cannot be iterated over using oop->size(). The only way to safely iterate over those is using

@@ -784,17 +783,12 @@
   void work(uint worker_id) {
     ShenandoahHeapRegion* region = _regions.next();
     ShenandoahHeap* heap = ShenandoahHeap::heap();
     ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
     while (region != NULL) {
-      if (heap->is_bitmap_slice_committed(region) && !region->is_pinned()) {
-        HeapWord* bottom = region->bottom();
-        HeapWord* top = ctx->top_at_mark_start(region->region_number());
-        if (top > bottom && region->has_live()) {
-          ctx->clear_bitmap(bottom, top);
-        }
-        assert(ctx->is_bitmap_clear_range(bottom, region->end()), "must be clear");
+      if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
+        ctx->clear_bitmap(region);
       }
       region = _regions.next();
     }
   }
 };

@@ -827,22 +821,15 @@
 
   // Bring regions in proper states after the collection, and set heap properties.
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
 
-    ShenandoahHeapLocker lock(heap->lock());
     ShenandoahPostCompactClosure post_compact;
     heap->heap_region_iterate(&post_compact);
     heap->set_used(post_compact.get_live());
 
     heap->collection_set()->clear();
     heap->free_set()->rebuild();
   }
 
   heap->clear_cancelled_gc();
-
-  // Also clear the next bitmap in preparation for next marking.
-  {
-    ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_next);
-    heap->reset_next_mark_bitmap();
-  }
 }