src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp

rev 59811 : 8247670: Shenandoah: deadlock during class unloading OOME

@@ -218,10 +218,11 @@
 
 class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
 private:
   bool                      _unloading_occurred;
   volatile bool             _failed;
+  bool                      _refill_icBuffer;
   ShenandoahHeap* const     _heap;
   BarrierSetNMethod* const  _bs;
 
   void set_failed() {
     Atomic::store(&_failed, true);

@@ -245,10 +246,11 @@
    }
 public:
   ShenandoahNMethodUnlinkClosure(bool unloading_occurred) :
       _unloading_occurred(unloading_occurred),
       _failed(false),
+      _refill_icBuffer(false),
       _heap(ShenandoahHeap::heap()),
       _bs(ShenandoahBarrierSet::barrier_set()->barrier_set_nmethod()) {}
 
   virtual void do_nmethod(nmethod* nm) {
     assert(_heap->is_concurrent_weak_root_in_progress(), "Only this phase");

@@ -262,29 +264,42 @@
     if (!nm->is_alive()) {
       return;
     }
 
     if (nm->is_unloading()) {
-      ShenandoahReentrantLocker locker(nm_data->lock());
+      ShenandoahAbortableNMethodLocker locker(nm_data->lock());
+      if (locker.aborted()) {
+        set_failed();
+      } else {
       unlink(nm);
+      }
       return;
     }
 
-    ShenandoahReentrantLocker locker(nm_data->lock());
+    ShenandoahAbortableNMethodLocker locker(nm_data->lock());
+    if (locker.aborted()) {
+      set_failed();
+      return;
+    }
 
     // Heal oops and disarm
     if (_bs->is_armed(nm)) {
       ShenandoahNMethod::heal_nmethod_metadata(nm_data);
       _bs->disarm(nm);
     }
 
     // Clear compiled ICs and exception caches
     if (!nm->unload_nmethod_caches(_unloading_occurred)) {
+      _refill_icBuffer = true;
       set_failed();
     }
   }
 
+  bool need_refill_icBuffer() const {
+    return _refill_icBuffer;
+  }
+
   bool failed() const {
     return Atomic::load(&_failed);
   }
 };
 

@@ -316,33 +331,41 @@
   }
 
   bool success() const {
     return !_cl.failed();
   }
+
+  bool need_refill_icBuffer() const {
+    return _cl.need_refill_icBuffer();
+  }
 };
 
 void ShenandoahCodeRoots::unlink(WorkGang* workers, bool unloading_occurred) {
   assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
          "Only when running concurrent class unloading");
 
   for (;;) {
     ICRefillVerifier verifier;
+    bool need_refill_icBuffer = false;
 
     {
       ShenandoahUnlinkTask task(unloading_occurred, &verifier);
       workers->run_task(&task);
       if (task.success()) {
         return;
       }
+      need_refill_icBuffer = task.need_refill_icBuffer();
     }
 
+    if (need_refill_icBuffer) {
     // Cleaning failed because we ran out of transitional IC stubs,
     // so we have to refill and try again. Refilling requires taking
     // a safepoint, so we temporarily leave the suspendible thread set.
     SuspendibleThreadSetLeaver sts;
     InlineCacheBuffer::refill_ic_stubs();
   }
+  }
 }
 
 class ShenandoahNMethodPurgeClosure : public NMethodClosure {
 public:
   virtual void do_nmethod(nmethod* nm) {