src/share/classes/java/lang/ref/Reference.java

*** 24,33 ****
--- 24,34 ----
   */
  
  package java.lang.ref;
  
  import sun.misc.Cleaner;
+ import sun.misc.Unsafe;
  
  /**
   * Abstract base class for reference objects.  This class defines the
   * operations common to all reference objects.  Because reference objects are
   * implemented in close cooperation with the garbage collector, this class may
*** 124,171 ****
  
      /* High-priority thread to enqueue pending References
       */
      private static class ReferenceHandler extends Thread {
  
          ReferenceHandler(ThreadGroup g, String name) {
              super(g, name);
          }
  
          public void run() {
              for (;;) {
                  Reference<Object> r;
                  synchronized (lock) {
-                     if (pending != null) {
                          r = pending;
                          pending = r.discovered;
                          r.discovered = null;
                      } else {
                          // The waiting on the lock may cause an OOME because it may try to allocate
!                         // exception objects, so also catch OOME here to avoid silent exit of the
!                         // reference handler thread.
!                         //
!                         // Explicitly define the order of the two exceptions we catch here
!                         // when waiting for the lock.
!                         //
!                         // We do not want to try to potentially load the InterruptedException class
!                         // (which would be done if this was its first use, and InterruptedException
!                         // were checked first) in this situation.
!                         //
!                         // This may lead to the VM not ever trying to load the InterruptedException
!                         // class again.
!                         try {
!                             try {
                                  lock.wait();
-                             } catch (OutOfMemoryError x) { }
-                         } catch (InterruptedException x) { }
                          continue;
                      }
                  }
  
                  // Fast path for cleaners
!                 if (r instanceof Cleaner) {
!                     ((Cleaner)r).clean();
                      continue;
                  }
  
                  ReferenceQueue<Object> q = r.queue;
                  if (q != ReferenceQueue.NULL) q.enqueue(r);
--- 125,185 ----
  
      /* High-priority thread to enqueue pending References
       */
      private static class ReferenceHandler extends Thread {
  
+         static {
+             // pre-load and initialize InterruptedException and Cleaner classes
+             // so that we don't get into trouble later in the run loop if there's
+             // memory shortage while loading/initializing them lazily.
+             Unsafe unsafe = Unsafe.getUnsafe();
+             unsafe.ensureClassInitialized(InterruptedException.class);
+             unsafe.ensureClassInitialized(Cleaner.class);
+         }
+ 
          ReferenceHandler(ThreadGroup g, String name) {
              super(g, name);
          }
  
          public void run() {
              for (;;) {
                  Reference<Object> r;
+                 Cleaner c;
+                 try {
                      synchronized (lock) {
                          r = pending;
+                         if (r != null) {
+                             // 'instanceof' might throw OOME sometimes so do this before
+                             // unlinking 'r' from the 'pending' chain...
+                             c = r instanceof Cleaner ? (Cleaner) r : null;
+                             // unlink 'r' from 'pending' chain
                              pending = r.discovered;
                              r.discovered = null;
                          } else {
                              // The waiting on the lock may cause an OOME because it may try to allocate
!                             // exception objects.
                              lock.wait();
                              continue;
                          }
                      }
+                 } catch (OutOfMemoryError x) {
+                     // Catch OOME from 'r instanceof Cleaner' or 'lock.wait()'.
+                     // Give other threads CPU time so they hopefully drop some live references
+                     // and GC reclaims some space.
+                     // Also prevent CPU intensive spinning in case 'r instanceof Cleaner' above
+                     // persistently throws OOME for some time...
+                     Thread.yield();
+                     // retry
+                     continue;
+                 } catch (InterruptedException x) {
+                     // Catch InterruptedException from 'lock.wait()' and retry
+                     continue;
+                 }
  
                  // Fast path for cleaners
!                 if (c != null) {
!                     c.clean();
                      continue;
                  }
  
                  ReferenceQueue<Object> q = r.queue;
                  if (q != ReferenceQueue.NULL) q.enqueue(r);