src/java.base/share/classes/java/nio/Bits.java

@@ -24,16 +24,18 @@
  */
 
 package java.nio;
 
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.StampedLock;
 
 import jdk.internal.misc.JavaNioAccess;
 import jdk.internal.misc.JavaLangRefAccess;
 import jdk.internal.misc.SharedSecrets;
 import jdk.internal.misc.Unsafe;
 import jdk.internal.misc.VM;
+import jdk.internal.ref.CleanerFactory;
 
 /**
  * Access to bits, native and otherwise.
  */
 

@@ -601,15 +603,18 @@
     private static final AtomicLong reservedMemory = new AtomicLong();
     private static final AtomicLong totalCapacity = new AtomicLong();
     private static final AtomicLong count = new AtomicLong();
     private static volatile boolean memoryLimitSet;
 
-    // max. number of sleeps during try-reserving with exponentially
-    // increasing delay before throwing OutOfMemoryError:
-    // 1, 2, 4, 8, 16, 32, 64, 128, 256 (total 511 ms ~ 0.5 s)
-    // which means that OOME will be thrown after 0.5 s of trying
-    private static final int MAX_SLEEPS = 9;
+    // A fair lock for direct memory allocator threads to queue after 1st optimistic
+    // reservation fails so that only a single thread at a time is retrying
+    // reservation while:
+    //   - helping out the Cleaner thread process enqueued Cleanable(s)
+    //   - followed by triggering reference discovery and waiting for references to be enqueued
+    //   - followed by another round of helping the Cleaner thread
+    // ... before finally giving up with OutOfMemoryError.
+    private static final StampedLock reservationLock = new StampedLock();
 
     // These methods should be called whenever direct memory is allocated or
     // freed.  They allow the user to control the amount of direct memory
     // which a process may access.  All sizes are specified in bytes.
     static void reserveMemory(long size, int cap) {

@@ -618,58 +623,58 @@
             maxMemory = VM.maxDirectMemory();
             memoryLimitSet = true;
         }
 
         // optimist!
+        long stamp = reservationLock.tryReadLock();
+        if (stamp != 0L) try {
         if (tryReserveMemory(size, cap)) {
             return;
         }
-
-        final JavaLangRefAccess jlra = SharedSecrets.getJavaLangRefAccess();
-
-        // retry while helping enqueue pending Reference objects
-        // which includes executing pending Cleaner(s) which includes
-        // Cleaner(s) that free direct buffer memory
-        while (jlra.tryHandlePendingReference()) {
-            if (tryReserveMemory(size, cap)) {
-                return;
-            }
+        } finally {
+            reservationLock.unlockRead(stamp);
         }
 
-        // trigger VM's Reference processing
-        System.gc();
-
-        // a retry loop with exponential back-off delays
-        // (this gives VM some time to do it's job)
+        // reservation threads that don't succeed at first must queue so that
+        // some of them don't starve while others succeed.
         boolean interrupted = false;
+        stamp = reservationLock.writeLock();
         try {
-            long sleepTime = 1;
-            int sleeps = 0;
-            while (true) {
+            JavaLangRefAccess jlra = SharedSecrets.getJavaLangRefAccess();
+
+            // retry reservation while helping the Cleaner thread process enqueued
+            // Cleanable(s) which includes the ones that free direct buffer memory
+            // until the queue drains out
+            do {
                 if (tryReserveMemory(size, cap)) {
                     return;
                 }
-                if (sleeps >= MAX_SLEEPS) {
-                    break;
-                }
-                if (!jlra.tryHandlePendingReference()) {
+            } while (jlra.cleanNextEnqueuedCleanable(CleanerFactory.cleaner()));
+
+            // trigger Reference discovery and wait until discovered Reference(s)
+            // have been enqueued...
                     try {
-                        Thread.sleep(sleepTime);
-                        sleepTime <<= 1;
-                        sleeps++;
+                jlra.discoverAndEnqueueReferences();
                     } catch (InterruptedException e) {
+                // don't swallow interrupts
                         interrupted = true;
                     }
+
+            // retry reservation while helping the Cleaner thread process enqueued
+            // Cleanable(s) which includes the ones that free direct buffer memory
+            // until the queue drains out
+            do {
+                if (tryReserveMemory(size, cap)) {
+                    return;
                 }
-            }
+            } while (jlra.cleanNextEnqueuedCleanable(CleanerFactory.cleaner()));
 
             // no luck
             throw new OutOfMemoryError("Direct buffer memory");
-
         } finally {
+            reservationLock.unlockWrite(stamp);
             if (interrupted) {
-                // don't swallow interrupts
                 Thread.currentThread().interrupt();
             }
         }
     }