src/java.base/share/classes/java/lang/ref/Reference.java

@@ -171,18 +171,14 @@
      *         or thread was interrupted before being notified;
      *         {@code false} otherwise.
      */
     static boolean tryHandlePending(boolean waitForNotify) {
         Reference<Object> r;
-        Cleaner c;
         try {
             synchronized (lock) {
                 if (pending != null) {
                     r = pending;
-                    // 'instanceof' might throw OutOfMemoryError sometimes
-                    // so do this before un-linking 'r' from the 'pending' chain...
-                    c = r instanceof Cleaner ? (Cleaner) r : null;
                     // unlink 'r' from 'pending' chain
                     pending = r.discovered;
                     r.discovered = null;
                 } else {
                     // The waiting on the lock may cause an OutOfMemoryError

@@ -195,20 +191,39 @@
                 }
             }
         } catch (OutOfMemoryError x) {
             // Give other threads CPU time so they hopefully drop some live references
             // and GC reclaims some space.
-            // Also prevent CPU intensive spinning in case 'r instanceof Cleaner' above
-            // persistently throws OOME for some time...
             Thread.yield();
             // retry
             return true;
         } catch (InterruptedException x) {
             // retry
             return true;
         }
 
+        Cleaner c;
+        try {
+            // It was established that "instanceof Cleaner" can cause OOME
+            // to be thrown. The only logical explanation is that this is caused
+            // by unsuccessful loading of the Cleaner class, triggered by instanceof.
+            // So now we pre-load Cleaner class at Reference class initialization
+            // time, but we are superstitious and still catch OOME here.
+            c = (r instanceof Cleaner) ? (Cleaner) r : null;
+        } catch (OutOfMemoryError x) {
+            // Give other threads CPU time so they hopefully drop some live references
+            // and GC reclaims some space.
+            Thread.yield();
+            // re-link 'r' back to 'pending' chain
+            synchronized (lock) {
+                r.discovered = pending;
+                pending = r;
+            }
+            // retry
+            return true;
+        }
+
         // Fast path for cleaners
         if (c != null) {
             c.clean();
             return true;
         }