src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

rev 55336 : 8225550: Shenandoah: Prevent SH::object_iterate() call's side-effects
rev 55337 : 8225573: Shenandoah: Enhance ShenandoahVerifier to ensure roots to-space invariant
rev 55339 : 8225582: Shenandoah: Enable concurrent evacuation of JNIHandles and CLDG roots

*** 36,45 ****
--- 36,46 ----
  #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
+ #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  #include "gc/shenandoah/shenandoahControlThread.hpp"
  #include "gc/shenandoah/shenandoahFreeSet.hpp"
  #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  #include "gc/shenandoah/shenandoahHeapRegion.hpp"
*** 947,957 ****
    {}
  
    void work(uint worker_id) {
      if (_concurrent) {
        ShenandoahConcurrentWorkerSession worker_session(worker_id);
!       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
        ShenandoahEvacOOMScope oom_evac_scope;
        do_work();
      } else {
        ShenandoahParallelWorkerSession worker_session(worker_id);
        ShenandoahEvacOOMScope oom_evac_scope;
--- 948,958 ----
    {}
  
    void work(uint worker_id) {
      if (_concurrent) {
        ShenandoahConcurrentWorkerSession worker_session(worker_id);
!       ShenandoahSuspendibleThreadSetJoiner stsj;
        ShenandoahEvacOOMScope oom_evac_scope;
        do_work();
      } else {
        ShenandoahParallelWorkerSession worker_session(worker_id);
        ShenandoahEvacOOMScope oom_evac_scope;
*** 1069,1081 ****
  void ShenandoahHeap::evacuate_and_update_roots() {
  #if COMPILER2_OR_JVMCI
    DerivedPointerTable::clear();
  #endif
    assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
- 
    {
!     ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
      ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
      workers()->run_task(&roots_task);
    }
  
  #if COMPILER2_OR_JVMCI
--- 1070,1084 ----
  void ShenandoahHeap::evacuate_and_update_roots() {
  #if COMPILER2_OR_JVMCI
    DerivedPointerTable::clear();
  #endif
    assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
    {
!     // Include concurrent roots if current cycle can not process those roots concurrently
!     bool include_concurrent_roots = !ShenandoahConcurrentRoots::should_do_concurrent_roots();
! 
!     ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac, include_concurrent_roots);
      ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
      workers()->run_task(&roots_task);
    }
  
  #if COMPILER2_OR_JVMCI
*** 1515,1525 ****
--- 1518,1533 ----
        if (ShenandoahPacing) {
          pacer()->setup_for_evac();
        }
  
        if (ShenandoahVerify) {
+         if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
+           ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::CLDGRoots);
+           verifier()->verify_roots_no_forwarded_except(types);
+         } else {
            verifier()->verify_roots_no_forwarded();
+         }
          verifier()->verify_during_evacuation();
        }
      } else {
        if (ShenandoahVerify) {
          verifier()->verify_after_concmark();
*** 1576,1585 ****
--- 1584,1621 ----
  
  void ShenandoahHeap::op_cleanup() {
    free_set()->recycle_trash();
  }
  
+ class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
+ private:
+   SuspendibleThreadSetJoiner                            _sts_joiner;
+   ShenandoahJNIHandleRoots<true /*concurrent*/>         _jni_roots;
+   ShenandoahClassLoaderDataRoots<true /* concurrent */> _cld_roots;
+ public:
+   ShenandoahConcurrentRootsEvacUpdateTask() :
+     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task") {
+   }
+ 
+   void work(uint worker_id) {
+     ShenandoahEvacOOMScope oom;
+     ShenandoahEvacuateUpdateRootsClosure cl;
+     CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
+ 
+     _jni_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
+     _cld_roots.cld_do(&clds);
+   }
+ };
+ 
+ void ShenandoahHeap::op_concurrent_roots() {
+   if (is_evacuation_in_progress() &&
+       ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
+     ShenandoahConcurrentRootsEvacUpdateTask task;
+     workers()->run_task(&task);
+   }
+ }
+ 
  void ShenandoahHeap::op_reset() {
    reset_mark_bitmap();
  }
  
  void ShenandoahHeap::op_preclean() {
*** 1849,1859 ****
  bool ShenandoahHeap::try_cancel_gc() {
    while (true) {
      jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
      if (prev == CANCELLABLE) return true;
      else if (prev == CANCELLED) return false;
-     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
      assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
      {
        // We need to provide a safepoint here, otherwise we might
        // spin forever if a SP is pending.
        ThreadBlockInVM sp(JavaThread::current());
--- 1885,1894 ----
*** 2063,2073 ****
    }
  
    void work(uint worker_id) {
      if (_concurrent) {
        ShenandoahConcurrentWorkerSession worker_session(worker_id);
!       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
        do_work();
      } else {
        ShenandoahParallelWorkerSession worker_session(worker_id);
        do_work();
      }
--- 2098,2108 ----
    }
  
    void work(uint worker_id) {
      if (_concurrent) {
        ShenandoahConcurrentWorkerSession worker_session(worker_id);
!       ShenandoahSuspendibleThreadSetJoiner stsj;
        do_work();
      } else {
        ShenandoahParallelWorkerSession worker_session(worker_id);
        do_work();
      }
*** 2263,2281 ****
    }
    return true;
  }
  
  void ShenandoahHeap::safepoint_synchronize_begin() {
-   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
      SuspendibleThreadSet::synchronize();
-   }
  }
  
  void ShenandoahHeap::safepoint_synchronize_end() {
-   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
      SuspendibleThreadSet::desynchronize();
-   }
  }
  
  void ShenandoahHeap::vmop_entry_init_mark() {
    TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
    ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
--- 2298,2312 ----
*** 2536,2545 ****
--- 2567,2592 ----
                                "concurrent reference update");
  
    try_inject_alloc_failure();
    op_updaterefs();
  }
+ 
+ void ShenandoahHeap::entry_concurrent_roots() {
+   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_roots);
+ 
+   static const char* msg = "Concurrent roots processing";
+   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
+   EventMark em("%s", msg);
+ 
+   ShenandoahWorkerScope scope(workers(),
+                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
+                               "concurrent root processing");
+ 
+   try_inject_alloc_failure();
+   op_concurrent_roots();
+ }
+ 
  void ShenandoahHeap::entry_cleanup() {
    ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
  
    static const char* msg = "Concurrent cleanup";
    GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);