src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.cpp

rev 59811 : 8247670: Shenandoah: deadlock during class unloading OOME

@@ -35,18 +35,24 @@
 #include "memory/resourceArea.hpp"
 
 bool ShenandoahBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
   ShenandoahReentrantLock* lock = ShenandoahNMethod::lock_for_nmethod(nm);
   assert(lock != NULL, "Must be");
-  ShenandoahReentrantLocker locker(lock);
+  ShenandoahAbortableNMethodLocker locker(lock);
 
   if (!is_armed(nm)) {
     // Some other thread got here first and healed the oops
     // and disarmed the nmethod.
     return true;
   }
 
+  // We may not be able to acquire the lock without potential deadlock if
+  // we run into evacuation OOM, just return false
+  if (locker.aborted()) {
+    return false;
+  }
+
   if (nm->is_unloading()) {
     // We don't need to take the lock when unlinking nmethods from
     // the Method, because it is only concurrently unlinked by
     // the entry barrier, which acquires the per nmethod lock.
     nm->unlink_from_method();