src/share/classes/java/lang/ref/Reference.java

@@ -24,10 +24,11 @@
  */
 
 package java.lang.ref;
 
 import sun.misc.Cleaner;
+import sun.misc.Unsafe;
 
 /**
  * Abstract base class for reference objects.  This class defines the
  * operations common to all reference objects.  Because reference objects are
  * implemented in close cooperation with the garbage collector, this class may

@@ -124,48 +125,61 @@
 
     /* High-priority thread to enqueue pending References
      */
     private static class ReferenceHandler extends Thread {
 
+        static {
+            // pre-load and initialize InterruptedException and Cleaner classes
+            // so that we don't get into trouble later in the run loop if there's
+            // memory shortage while loading/initializing them lazily.
+            Unsafe unsafe = Unsafe.getUnsafe();
+            unsafe.ensureClassInitialized(InterruptedException.class);
+            unsafe.ensureClassInitialized(Cleaner.class);
+        }
+
         ReferenceHandler(ThreadGroup g, String name) {
             super(g, name);
         }
 
         public void run() {
             for (;;) {
                 Reference<Object> r;
+                Cleaner c;
+                try {
                 synchronized (lock) {
-                    if (pending != null) {
                         r = pending;
+                        if (r != null) {
+                            // 'instanceof' might throw OOME sometimes so do this before
+                            // unlinking 'r' from the 'pending' chain...
+                            c = r instanceof Cleaner ? (Cleaner) r : null;
+                            // unlink 'r' from 'pending' chain
                         pending = r.discovered;
                         r.discovered = null;
                     } else {
                         // The waiting on the lock may cause an OOME because it may try to allocate
-                        // exception objects, so also catch OOME here to avoid silent exit of the
-                        // reference handler thread.
-                        //
-                        // Explicitly define the order of the two exceptions we catch here
-                        // when waiting for the lock.
-                        //
-                        // We do not want to try to potentially load the InterruptedException class
-                        // (which would be done if this was its first use, and InterruptedException
-                        // were checked first) in this situation.
-                        //
-                        // This may lead to the VM not ever trying to load the InterruptedException
-                        // class again.
-                        try {
-                            try {
+                            // exception objects.
                                 lock.wait();
-                            } catch (OutOfMemoryError x) { }
-                        } catch (InterruptedException x) { }
                         continue;
                     }
                 }
+                } catch (OutOfMemoryError x) {
+                    // Catch OOME from 'r instanceof Cleaner' or 'lock.wait()'.
+                    // Give other threads CPU time so they hopefully drop some live references
+                    // and GC reclaims some space.
+                    // Also prevent CPU intensive spinning in case 'r instanceof Cleaner' above
+                    // persistently throws OOME for some time...
+                    Thread.yield();
+                    // retry
+                    continue;
+                } catch (InterruptedException x) {
+                    // Catch InterruptedException from 'lock.wait()' and retry
+                    continue;
+                }
 
                 // Fast path for cleaners
-                if (r instanceof Cleaner) {
-                    ((Cleaner)r).clean();
+                if (c != null) {
+                    c.clean();
                     continue;
                 }
 
                 ReferenceQueue<Object> q = r.queue;
                 if (q != ReferenceQueue.NULL) q.enqueue(r);