src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

rev 55336 : 8225550: Shenandoah: Prevent SH::object_iterate() call's side-effects
rev 55337 : 8225573: Shenandoah: Enhance ShenandoahVerifier to ensure roots to-space invariant
rev 55339 : 8225582: Shenandoah: Enable concurrent evacuation of JNIHandles and CLDG roots

@@ -36,10 +36,11 @@
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
+#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahControlThread.hpp"
 #include "gc/shenandoah/shenandoahFreeSet.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"

@@ -947,11 +948,11 @@
   {}
 
   void work(uint worker_id) {
     if (_concurrent) {
       ShenandoahConcurrentWorkerSession worker_session(worker_id);
-      ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
+      ShenandoahSuspendibleThreadSetJoiner stsj;
       ShenandoahEvacOOMScope oom_evac_scope;
       do_work();
     } else {
       ShenandoahParallelWorkerSession worker_session(worker_id);
       ShenandoahEvacOOMScope oom_evac_scope;

@@ -1069,13 +1070,15 @@
 void ShenandoahHeap::evacuate_and_update_roots() {
 #if COMPILER2_OR_JVMCI
   DerivedPointerTable::clear();
 #endif
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
-
   {
-    ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
+    // Include concurrent roots if current cycle can not process those roots concurrently
+    bool include_concurrent_roots = !ShenandoahConcurrentRoots::should_do_concurrent_roots();
+
+    ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac, include_concurrent_roots);
     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
     workers()->run_task(&roots_task);
   }
 
 #if COMPILER2_OR_JVMCI

@@ -1515,11 +1518,16 @@
       if (ShenandoahPacing) {
         pacer()->setup_for_evac();
       }
 
       if (ShenandoahVerify) {
+        if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
+          ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::CLDGRoots);
+          verifier()->verify_roots_no_forwarded_except(types);
+        } else {
         verifier()->verify_roots_no_forwarded();
+        }
         verifier()->verify_during_evacuation();
       }
     } else {
       if (ShenandoahVerify) {
         verifier()->verify_after_concmark();

@@ -1576,10 +1584,38 @@
 
 void ShenandoahHeap::op_cleanup() {
   free_set()->recycle_trash();
 }
 
+class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
+private:
+  SuspendibleThreadSetJoiner                            _sts_joiner;
+  ShenandoahJNIHandleRoots<true /*concurrent*/>         _jni_roots;
+  ShenandoahClassLoaderDataRoots<true /* concurrent */> _cld_roots;
+public:
+  ShenandoahConcurrentRootsEvacUpdateTask() :
+    AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task") {
+  }
+
+  void work(uint worker_id) {
+    ShenandoahEvacOOMScope oom;
+    ShenandoahEvacuateUpdateRootsClosure cl;
+    CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
+
+    _jni_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
+    _cld_roots.cld_do(&clds);
+  }
+};
+
+void ShenandoahHeap::op_concurrent_roots() {
+  if (is_evacuation_in_progress() &&
+      ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
+    ShenandoahConcurrentRootsEvacUpdateTask task;
+    workers()->run_task(&task);
+  }
+}
+
 void ShenandoahHeap::op_reset() {
   reset_mark_bitmap();
 }
 
 void ShenandoahHeap::op_preclean() {

@@ -1849,11 +1885,10 @@
 bool ShenandoahHeap::try_cancel_gc() {
   while (true) {
     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
     if (prev == CANCELLABLE) return true;
     else if (prev == CANCELLED) return false;
-    assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
     {
       // We need to provide a safepoint here, otherwise we might
       // spin forever if a SP is pending.
       ThreadBlockInVM sp(JavaThread::current());

@@ -2063,11 +2098,11 @@
   }
 
   void work(uint worker_id) {
     if (_concurrent) {
       ShenandoahConcurrentWorkerSession worker_session(worker_id);
-      ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
+      ShenandoahSuspendibleThreadSetJoiner stsj;
       do_work();
     } else {
       ShenandoahParallelWorkerSession worker_session(worker_id);
       do_work();
     }

@@ -2263,19 +2298,15 @@
   }
   return true;
 }
 
 void ShenandoahHeap::safepoint_synchronize_begin() {
-  if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
     SuspendibleThreadSet::synchronize();
-  }
 }
 
 void ShenandoahHeap::safepoint_synchronize_end() {
-  if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
     SuspendibleThreadSet::desynchronize();
-  }
 }
 
 void ShenandoahHeap::vmop_entry_init_mark() {
   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);

@@ -2536,10 +2567,26 @@
                               "concurrent reference update");
 
   try_inject_alloc_failure();
   op_updaterefs();
 }
+
+void ShenandoahHeap::entry_concurrent_roots() {
+  ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_roots);
+
+  static const char* msg = "Concurrent roots processing";
+  GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
+  EventMark em("%s", msg);
+
+  ShenandoahWorkerScope scope(workers(),
+                              ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
+                              "concurrent root processing");
+
+  try_inject_alloc_failure();
+  op_concurrent_roots();
+}
+
 void ShenandoahHeap::entry_cleanup() {
   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
 
   static const char* msg = "Concurrent cleanup";
   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);