1 /*
   2  * Copyright (c) 1996, 2012, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package sun.rmi.transport;
  26 
  27 import java.lang.ref.PhantomReference;
  28 import java.lang.ref.ReferenceQueue;
  29 import java.security.AccessController;
  30 import java.security.PrivilegedAction;
  31 import java.util.HashMap;
  32 import java.util.HashSet;
  33 import java.util.Iterator;
  34 import java.util.List;
  35 import java.util.Map;
  36 import java.util.Set;
  37 import java.rmi.ConnectException;
  38 import java.rmi.RemoteException;
  39 import java.rmi.dgc.DGC;
  40 import java.rmi.dgc.Lease;
  41 import java.rmi.dgc.VMID;
  42 import java.rmi.server.ObjID;
  43 import sun.misc.GC;
  44 import sun.rmi.runtime.NewThreadAction;
  45 import sun.rmi.server.UnicastRef;
  46 import sun.rmi.server.Util;
  47 
  48 /**
  49  * DGCClient implements the client-side of the RMI distributed garbage
  50  * collection system.
  51  *
  52  * The external interface to DGCClient is the "registerRefs" method.
  53  * When a LiveRef to a remote object enters the VM, it needs to be
  54  * registered with the DGCClient to participate in distributed garbage
  55  * collection.
  56  *
  57  * When the first LiveRef to a particular remote object is registered,
  58  * a "dirty" call is made to the server-side distributed garbage
  59  * collector for the remote object, which returns a lease guaranteeing
  60  * that the server-side DGC will not collect the remote object for a
  61  * certain period of time.  While LiveRef instances to remote objects
  62  * on a particular server exist, the DGCClient periodically sends more
  63  * "dirty" calls to renew its lease.
  64  *
  65  * The DGCClient tracks the local reachability of registered LiveRef
  66  * instances (using phantom references).  When the LiveRef instance
  67  * for a particular remote object becomes garbage collected locally,
  68  * a "clean" call is made to the server-side distributed garbage
  69  * collector, indicating that the server no longer needs to keep the
  70  * remote object alive for this client.
  71  *
  72  * @see java.rmi.dgc.DGC, sun.rmi.transport.DGCImpl
  73  *
  74  * @author  Ann Wollrath
  75  * @author  Peter Jones
  76  */
  77 final class DGCClient {
  78 
  79     /** next sequence number for DGC calls (access synchronized on class) */
  80     private static long nextSequenceNum = Long.MIN_VALUE;
  81 
  82     /** unique identifier for this VM as a client of DGC */
  83     private static VMID vmid = new VMID();
  84 
  85     /** lease duration to request (usually ignored by server) */
  86     private static final long leaseValue =              // default 10 minutes
  87         AccessController.doPrivileged((PrivilegedAction<Long>) () ->
  88             Long.getLong("java.rmi.dgc.leaseValue", 600000));
  89 
  90     /** maximum interval between retries of failed clean calls */
  91     private static final long cleanInterval =           // default 3 minutes
  92         AccessController.doPrivileged((PrivilegedAction<Long>) () ->
  93             Long.getLong("sun.rmi.dgc.cleanInterval", 180000));
  94 
  95     /** maximum interval between complete garbage collections of local heap */
  96     private static final long gcInterval =              // default 1 hour
  97         AccessController.doPrivileged((PrivilegedAction<Long>) () ->
  98             Long.getLong("sun.rmi.dgc.client.gcInterval", 3600000));
  99 
 100     /** minimum retry count for dirty calls that fail */
 101     private static final int dirtyFailureRetries = 5;
 102 
 103     /** retry count for clean calls that fail with ConnectException */
 104     private static final int cleanFailureRetries = 5;
 105 
 106     /** constant empty ObjID array for lease renewal optimization */
 107     private static final ObjID[] emptyObjIDArray = new ObjID[0];
 108 
 109     /** ObjID for server-side DGC object */
 110     private static final ObjID dgcID = new ObjID(ObjID.DGC_ID);
 111 
 112     /*
 113      * Disallow anyone from creating one of these.
 114      */
 115     private DGCClient() {}
 116 
 117     /**
 118      * Register the LiveRef instances in the supplied list to participate
 119      * in distributed garbage collection.
 120      *
 121      * All of the LiveRefs in the list must be for remote objects at the
 122      * given endpoint.
 123      */
 124     static void registerRefs(Endpoint ep, List<LiveRef> refs) {
 125         /*
 126          * Look up the given endpoint and register the refs with it.
 127          * The retrieved entry may get removed from the global endpoint
 128          * table before EndpointEntry.registerRefs() is able to acquire
 129          * its lock; in this event, it returns false, and we loop and
 130          * try again.
 131          */
 132         EndpointEntry epEntry;
 133         do {
 134             epEntry = EndpointEntry.lookup(ep);
 135         } while (!epEntry.registerRefs(refs));
 136     }
 137 
 138     /**
 139      * Get the next sequence number to be used for a dirty or clean
 140      * operation from this VM.  This method should only be called while
 141      * synchronized on the EndpointEntry whose data structures the
 142      * operation affects.
 143      */
 144     private static synchronized long getNextSequenceNum() {
 145         return nextSequenceNum++;
 146     }
 147 
 148     /**
 149      * Given the length of a lease and the time that it was granted,
 150      * compute the absolute time at which it should be renewed, giving
 151      * room for reasonable computational and communication delays.
 152      */
 153     private static long computeRenewTime(long grantTime, long duration) {
 154         /*
 155          * REMIND: This algorithm should be more sophisticated, waiting
 156          * a longer fraction of the lease duration for longer leases.
 157          */
 158         return grantTime + (duration / 2);
 159     }
 160 
 161     /**
 162      * EndpointEntry encapsulates the client-side DGC information specific
 163      * to a particular Endpoint.  Of most significance is the table that
 164      * maps LiveRef value to RefEntry objects and the renew/clean thread
 165      * that handles asynchronous client-side DGC operations.
 166      */
 167     private static class EndpointEntry {
 168 
 169         /** the endpoint that this entry is for */
 170         private Endpoint endpoint;
 171         /** synthesized reference to the remote server-side DGC */
 172         private DGC dgc;
 173 
 174         /** table of refs held for endpoint: maps LiveRef to RefEntry */
 175         private Map<LiveRef, RefEntry> refTable = new HashMap<>(5);
 176         /** set of RefEntry instances from last (failed) dirty call */
 177         private Set<RefEntry> invalidRefs = new HashSet<>(5);
 178 
 179         /** true if this entry has been removed from the global table */
 180         private boolean removed = false;
 181 
 182         /** absolute time to renew current lease to this endpoint */
 183         private long renewTime = Long.MAX_VALUE;
 184         /** absolute time current lease to this endpoint will expire */
 185         private long expirationTime = Long.MIN_VALUE;
 186         /** count of recent dirty calls that have failed */
 187         private int dirtyFailures = 0;
 188         /** absolute time of first recent failed dirty call */
 189         private long dirtyFailureStartTime;
 190         /** (average) elapsed time for recent failed dirty calls */
 191         private long dirtyFailureDuration;
 192 
 193         /** renew/clean thread for handling lease renewals and clean calls */
 194         private Thread renewCleanThread;
 195         /** true if renew/clean thread may be interrupted */
 196         private boolean interruptible = false;
 197 
 198         /** reference queue for phantom references */
 199         private ReferenceQueue<LiveRef> refQueue = new ReferenceQueue<>();
 200         /** set of clean calls that need to be made */
 201         private Set<CleanRequest> pendingCleans = new HashSet<>(5);
 202 
 203         /** global endpoint table: maps Endpoint to EndpointEntry */
 204         private static Map<Endpoint,EndpointEntry> endpointTable = new HashMap<>(5);
 205         /** handle for GC latency request (for future cancellation) */
 206         private static GC.LatencyRequest gcLatencyRequest = null;
 207 
 208         /**
 209          * Look up the EndpointEntry for the given Endpoint.  An entry is
 210          * created if one does not already exist.
 211          */
 212         public static EndpointEntry lookup(Endpoint ep) {
 213             synchronized (endpointTable) {
 214                 EndpointEntry entry = endpointTable.get(ep);
 215                 if (entry == null) {
 216                     entry = new EndpointEntry(ep);
 217                     endpointTable.put(ep, entry);
 218                     /*
 219                      * While we are tracking live remote references registered
 220                      * in this VM, request a maximum latency for inspecting the
 221                      * entire heap from the local garbage collector, to place
 222                      * an upper bound on the time to discover remote references
 223                      * that have become unreachable (see bugid 4171278).
 224                      */
 225                     if (gcLatencyRequest == null) {
 226                         gcLatencyRequest = GC.requestLatency(gcInterval);
 227                     }
 228                 }
 229                 return entry;
 230             }
 231         }
 232 
 233         private EndpointEntry(final Endpoint endpoint) {
 234             this.endpoint = endpoint;
 235             try {
 236                 LiveRef dgcRef = new LiveRef(dgcID, endpoint, false);
 237                 dgc = (DGC) Util.createProxy(DGCImpl.class,
 238                                              new UnicastRef(dgcRef), true);
 239             } catch (RemoteException e) {
 240                 throw new Error("internal error creating DGC stub");
 241             }
 242             renewCleanThread =  AccessController.doPrivileged(
 243                 new NewThreadAction(new RenewCleanThread(),
 244                                     "RenewClean-" + endpoint, true));
 245             renewCleanThread.start();
 246         }
 247 
 248         /**
 249          * Register the LiveRef instances in the supplied list to participate
 250          * in distributed garbage collection.
 251          *
 252          * This method returns false if this entry was removed from the
 253          * global endpoint table (because it was empty) before these refs
 254          * could be registered.  In that case, a new EndpointEntry needs
 255          * to be looked up.
 256          *
 257          * This method must NOT be called while synchronized on this entry.
 258          */
 259         public boolean registerRefs(List<LiveRef> refs) {
 260             assert !Thread.holdsLock(this);
 261 
 262             Set<RefEntry> refsToDirty = null;     // entries for refs needing dirty
 263             long sequenceNum;           // sequence number for dirty call
 264 
 265             synchronized (this) {
 266                 if (removed) {
 267                     return false;
 268                 }
 269 
 270                 Iterator<LiveRef> iter = refs.iterator();
 271                 while (iter.hasNext()) {
 272                     LiveRef ref = iter.next();
 273                     assert ref.getEndpoint().equals(endpoint);
 274 
 275                     RefEntry refEntry = refTable.get(ref);
 276                     if (refEntry == null) {
 277                         LiveRef refClone = (LiveRef) ref.clone();
 278                         refEntry = new RefEntry(refClone);
 279                         refTable.put(refClone, refEntry);
 280                         if (refsToDirty == null) {
 281                             refsToDirty = new HashSet<>(5);
 282                         }
 283                         refsToDirty.add(refEntry);
 284                     }
 285 
 286                     refEntry.addInstanceToRefSet(ref);
 287                 }
 288 
 289                 if (refsToDirty == null) {
 290                     return true;
 291                 }
 292 
 293                 refsToDirty.addAll(invalidRefs);
 294                 invalidRefs.clear();
 295 
 296                 sequenceNum = getNextSequenceNum();
 297             }
 298 
 299             makeDirtyCall(refsToDirty, sequenceNum);
 300             return true;
 301         }
 302 
 303         /**
 304          * Remove the given RefEntry from the ref table.  If that makes
 305          * the ref table empty, remove this entry from the global endpoint
 306          * table.
 307          *
 308          * This method must ONLY be called while synchronized on this entry.
 309          */
 310         private void removeRefEntry(RefEntry refEntry) {
 311             assert Thread.holdsLock(this);
 312             assert !removed;
 313             assert refTable.containsKey(refEntry.getRef());
 314 
 315             refTable.remove(refEntry.getRef());
 316             invalidRefs.remove(refEntry);
 317             if (refTable.isEmpty()) {
 318                 synchronized (endpointTable) {
 319                     endpointTable.remove(endpoint);
 320                     Transport transport = endpoint.getOutboundTransport();
 321                     transport.free(endpoint);
 322                     /*
 323                      * If there are no longer any live remote references
 324                      * registered, we are no longer concerned with the
 325                      * latency of local garbage collection here.
 326                      */
 327                     if (endpointTable.isEmpty()) {
 328                         assert gcLatencyRequest != null;
 329                         gcLatencyRequest.cancel();
 330                         gcLatencyRequest = null;
 331                     }
 332                     removed = true;
 333                 }
 334             }
 335         }
 336 
 337         /**
 338          * Make a DGC dirty call to this entry's endpoint, for the ObjIDs
 339          * corresponding to the given set of refs and with the given
 340          * sequence number.
 341          *
 342          * This method must NOT be called while synchronized on this entry.
 343          */
 344         private void makeDirtyCall(Set<RefEntry> refEntries, long sequenceNum) {
 345             assert !Thread.holdsLock(this);
 346 
 347             ObjID[] ids;
 348             if (refEntries != null) {
 349                 ids = createObjIDArray(refEntries);
 350             } else {
 351                 ids = emptyObjIDArray;
 352             }
 353 
 354             long startTime = System.currentTimeMillis();
 355             try {
 356                 Lease lease =
 357                     dgc.dirty(ids, sequenceNum, new Lease(vmid, leaseValue));
 358                 long duration = lease.getValue();
 359 
 360                 long newRenewTime = computeRenewTime(startTime, duration);
 361                 long newExpirationTime = startTime + duration;
 362 
 363                 synchronized (this) {
 364                     dirtyFailures = 0;
 365                     setRenewTime(newRenewTime);
 366                     expirationTime = newExpirationTime;
 367                 }
 368 
 369             } catch (Exception e) {
 370                 long endTime = System.currentTimeMillis();
 371 
 372                 synchronized (this) {
 373                     dirtyFailures++;
 374 
 375                     if (dirtyFailures == 1) {
 376                         /*
 377                          * If this was the first recent failed dirty call,
 378                          * reschedule another one immediately, in case there
 379                          * was just a transient network problem, and remember
 380                          * the start time and duration of this attempt for
 381                          * future calculations of the delays between retries.
 382                          */
 383                         dirtyFailureStartTime = startTime;
 384                         dirtyFailureDuration = endTime - startTime;
 385                         setRenewTime(endTime);
 386                     } else {
 387                         /*
 388                          * For each successive failed dirty call, wait for a
 389                          * (binary) exponentially increasing delay before
 390                          * retrying, to avoid network congestion.
 391                          */
 392                         int n = dirtyFailures - 2;
 393                         if (n == 0) {
 394                             /*
 395                              * Calculate the initial retry delay from the
 396                              * average time elapsed for each of the first
 397                              * two failed dirty calls.  The result must be
 398                              * at least 1000ms, to prevent a tight loop.
 399                              */
 400                             dirtyFailureDuration =
 401                                 Math.max((dirtyFailureDuration +
 402                                           (endTime - startTime)) >> 1, 1000);
 403                         }
 404                         long newRenewTime =
 405                             endTime + (dirtyFailureDuration << n);
 406 
 407                         /*
 408                          * Continue if the last known held lease has not
 409                          * expired, or else at least a fixed number of times,
 410                          * or at least until we've tried for a fixed amount
 411                          * of time (the default lease value we request).
 412                          */
 413                         if (newRenewTime < expirationTime ||
 414                             dirtyFailures < dirtyFailureRetries ||
 415                             newRenewTime < dirtyFailureStartTime + leaseValue)
 416                         {
 417                             setRenewTime(newRenewTime);
 418                         } else {
 419                             /*
 420                              * Give up: postpone lease renewals until next
 421                              * ref is registered for this endpoint.
 422                              */
 423                             setRenewTime(Long.MAX_VALUE);
 424                         }
 425                     }
 426 
 427                     if (refEntries != null) {
 428                         /*
 429                          * Add all of these refs to the set of refs for this
 430                          * endpoint that may be invalid (this VM may not be in
 431                          * the server's referenced set), so that we will
 432                          * attempt to explicitly dirty them again in the
 433                          * future.
 434                          */
 435                         invalidRefs.addAll(refEntries);
 436 
 437                         /*
 438                          * Record that a dirty call has failed for all of these
 439                          * refs, so that clean calls for them in the future
 440                          * will be strong.
 441                          */
 442                         Iterator<RefEntry> iter = refEntries.iterator();
 443                         while (iter.hasNext()) {
 444                             RefEntry refEntry = iter.next();
 445                             refEntry.markDirtyFailed();
 446                         }
 447                     }
 448 
 449                     /*
 450                      * If the last known held lease will have expired before
 451                      * the next renewal, all refs might be invalid.
 452                      */
 453                     if (renewTime >= expirationTime) {
 454                         invalidRefs.addAll(refTable.values());
 455                     }
 456                 }
 457             }
 458         }
 459 
 460         /**
 461          * Set the absolute time at which the lease for this entry should
 462          * be renewed.
 463          *
 464          * This method must ONLY be called while synchronized on this entry.
 465          */
 466         private void setRenewTime(long newRenewTime) {
 467             assert Thread.holdsLock(this);
 468 
 469             if (newRenewTime < renewTime) {
 470                 renewTime = newRenewTime;
 471                 if (interruptible) {
 472                     AccessController.doPrivileged(
 473                         new PrivilegedAction<Void>() {
 474                             public Void run() {
 475                             renewCleanThread.interrupt();
 476                             return null;
 477                         }
 478                     });
 479                 }
 480             } else {
 481                 renewTime = newRenewTime;
 482             }
 483         }
 484 
 485         /**
 486          * RenewCleanThread handles the asynchronous client-side DGC activity
 487          * for this entry: renewing the leases and making clean calls.
 488          */
 489         private class RenewCleanThread implements Runnable {
 490 
 491             public void run() {
 492                 do {
 493                     long timeToWait;
 494                     RefEntry.PhantomLiveRef phantom = null;
 495                     boolean needRenewal = false;
 496                     Set<RefEntry> refsToDirty = null;
 497                     long sequenceNum = Long.MIN_VALUE;
 498 
 499                     synchronized (EndpointEntry.this) {
 500                         /*
 501                          * Calculate time to block (waiting for phantom
 502                          * reference notifications).  It is the time until the
 503                          * lease renewal should be done, bounded on the low
 504                          * end by 1 ms so that the reference queue will always
 505                          * get processed, and if there are pending clean
 506                          * requests (remaining because some clean calls
 507                          * failed), bounded on the high end by the maximum
 508                          * clean call retry interval.
 509                          */
 510                         long timeUntilRenew =
 511                             renewTime - System.currentTimeMillis();
 512                         timeToWait = Math.max(timeUntilRenew, 1);
 513                         if (!pendingCleans.isEmpty()) {
 514                             timeToWait = Math.min(timeToWait, cleanInterval);
 515                         }
 516 
 517                         /*
 518                          * Set flag indicating that it is OK to interrupt this
 519                          * thread now, such as if a earlier lease renewal time
 520                          * is set, because we are only going to be blocking
 521                          * and can deal with interrupts.
 522                          */
 523                         interruptible = true;
 524                     }
 525 
 526                     try {
 527                         /*
 528                          * Wait for the duration calculated above for any of
 529                          * our phantom references to be enqueued.
 530                          */
 531                         phantom = (RefEntry.PhantomLiveRef)
 532                             refQueue.remove(timeToWait);
 533                     } catch (InterruptedException e) {
 534                     }
 535 
 536                     synchronized (EndpointEntry.this) {
 537                         /*
 538                          * Set flag indicating that it is NOT OK to interrupt
 539                          * this thread now, because we may be undertaking I/O
 540                          * operations that should not be interrupted (and we
 541                          * will not be blocking arbitrarily).
 542                          */
 543                         interruptible = false;
 544                         Thread.interrupted();   // clear interrupted state
 545 
 546                         /*
 547                          * If there was a phantom reference enqueued, process
 548                          * it and all the rest on the queue, generating
 549                          * clean requests as necessary.
 550                          */
 551                         if (phantom != null) {
 552                             processPhantomRefs(phantom);
 553                         }
 554 
 555                         /*
 556                          * Check if it is time to renew this entry's lease.
 557                          */
 558                         long currentTime = System.currentTimeMillis();
 559                         if (currentTime > renewTime) {
 560                             needRenewal = true;
 561                             if (!invalidRefs.isEmpty()) {
 562                                 refsToDirty = invalidRefs;
 563                                 invalidRefs = new HashSet<>(5);
 564                             }
 565                             sequenceNum = getNextSequenceNum();
 566                         }
 567                     }
 568 
 569                     if (needRenewal) {
 570                         makeDirtyCall(refsToDirty, sequenceNum);
 571                     }
 572 
 573                     if (!pendingCleans.isEmpty()) {
 574                         makeCleanCalls();
 575                     }
 576                 } while (!removed || !pendingCleans.isEmpty());
 577             }
 578         }
 579 
 580         /**
 581          * Process the notification of the given phantom reference and any
 582          * others that are on this entry's reference queue.  Each phantom
 583          * reference is removed from its RefEntry's ref set.  All ref
 584          * entries that have no more registered instances are collected
 585          * into up to two batched clean call requests: one for refs
 586          * requiring a "strong" clean call, and one for the rest.
 587          *
 588          * This method must ONLY be called while synchronized on this entry.
 589          */
 590         private void processPhantomRefs(RefEntry.PhantomLiveRef phantom) {
 591             assert Thread.holdsLock(this);
 592 
 593             Set<RefEntry> strongCleans = null;
 594             Set<RefEntry> normalCleans = null;
 595 
 596             do {
 597                 RefEntry refEntry = phantom.getRefEntry();
 598                 refEntry.removeInstanceFromRefSet(phantom);
 599                 if (refEntry.isRefSetEmpty()) {
 600                     if (refEntry.hasDirtyFailed()) {
 601                         if (strongCleans == null) {
 602                             strongCleans = new HashSet<>(5);
 603                         }
 604                         strongCleans.add(refEntry);
 605                     } else {
 606                         if (normalCleans == null) {
 607                             normalCleans = new HashSet<>(5);
 608                         }
 609                         normalCleans.add(refEntry);
 610                     }
 611                     removeRefEntry(refEntry);
 612                 }
 613             } while ((phantom =
 614                 (RefEntry.PhantomLiveRef) refQueue.poll()) != null);
 615 
 616             if (strongCleans != null) {
 617                 pendingCleans.add(
 618                     new CleanRequest(createObjIDArray(strongCleans),
 619                                      getNextSequenceNum(), true));
 620             }
 621             if (normalCleans != null) {
 622                 pendingCleans.add(
 623                     new CleanRequest(createObjIDArray(normalCleans),
 624                                      getNextSequenceNum(), false));
 625             }
 626         }
 627 
 628         /**
 629          * CleanRequest holds the data for the parameters of a clean call
 630          * that needs to be made.
 631          */
 632         private static class CleanRequest {
 633 
 634             final ObjID[] objIDs;
 635             final long sequenceNum;
 636             final boolean strong;
 637 
 638             /** how many times this request has failed */
 639             int failures = 0;
 640 
 641             CleanRequest(ObjID[] objIDs, long sequenceNum, boolean strong) {
 642                 this.objIDs = objIDs;
 643                 this.sequenceNum = sequenceNum;
 644                 this.strong = strong;
 645             }
 646         }
 647 
 648         /**
 649          * Make all of the clean calls described by the clean requests in
 650          * this entry's set of "pending cleans".  Clean requests for clean
 651          * calls that succeed are removed from the "pending cleans" set.
 652          *
 653          * This method must NOT be called while synchronized on this entry.
 654          */
 655         private void makeCleanCalls() {
 656             assert !Thread.holdsLock(this);
 657 
 658             Iterator<CleanRequest> iter = pendingCleans.iterator();
 659             while (iter.hasNext()) {
 660                 CleanRequest request = iter.next();
 661                 try {
 662                     dgc.clean(request.objIDs, request.sequenceNum, vmid,
 663                               request.strong);
 664                     iter.remove();
 665                 } catch (Exception e) {
 666                     /*
 667                      * Many types of exceptions here could have been
 668                      * caused by a transient failure, so try again a
 669                      * few times, but not forever.
 670                      */
 671                     if (++request.failures >= cleanFailureRetries) {
 672                         iter.remove();
 673                     }
 674                 }
 675             }
 676         }
 677 
 678         /**
 679          * Create an array of ObjIDs (needed for the DGC remote calls)
 680          * from the ids in the given set of refs.
 681          */
 682         private static ObjID[] createObjIDArray(Set<RefEntry> refEntries) {
 683             ObjID[] ids = new ObjID[refEntries.size()];
 684             Iterator<RefEntry> iter = refEntries.iterator();
 685             for (int i = 0; i < ids.length; i++) {
 686                 ids[i] = iter.next().getRef().getObjID();
 687             }
 688             return ids;
 689         }
 690 
 691         /**
 692          * RefEntry encapsulates the client-side DGC information specific
 693          * to a particular LiveRef value.  In particular, it contains a
 694          * set of phantom references to all of the instances of the LiveRef
 695          * value registered in the system (but not garbage collected
 696          * locally).
 697          */
 698         private class RefEntry {
 699 
 700             /** LiveRef value for this entry (not a registered instance) */
 701             private LiveRef ref;
 702             /** set of phantom references to registered instances */
 703             private Set<PhantomLiveRef> refSet = new HashSet<>(5);
 704             /** true if a dirty call containing this ref has failed */
 705             private boolean dirtyFailed = false;
 706 
 707             public RefEntry(LiveRef ref) {
 708                 this.ref = ref;
 709             }
 710 
 711             /**
 712              * Return the LiveRef value for this entry (not a registered
 713              * instance).
 714              */
 715             public LiveRef getRef() {
 716                 return ref;
 717             }
 718 
 719             /**
 720              * Add a LiveRef to the set of registered instances for this entry.
 721              *
 722              * This method must ONLY be invoked while synchronized on this
 723              * RefEntry's EndpointEntry.
 724              */
 725             public void addInstanceToRefSet(LiveRef ref) {
 726                 assert Thread.holdsLock(EndpointEntry.this);
 727                 assert ref.equals(this.ref);
 728 
 729                 /*
 730                  * Only keep a phantom reference to the registered instance,
 731                  * so that it can be garbage collected normally (and we can be
 732                  * notified when that happens).
 733                  */
 734                 refSet.add(new PhantomLiveRef(ref));
 735             }
 736 
 737             /**
 738              * Remove a PhantomLiveRef from the set of registered instances.
 739              *
 740              * This method must ONLY be invoked while synchronized on this
 741              * RefEntry's EndpointEntry.
 742              */
 743             public void removeInstanceFromRefSet(PhantomLiveRef phantom) {
 744                 assert Thread.holdsLock(EndpointEntry.this);
 745                 assert refSet.contains(phantom);
 746                 refSet.remove(phantom);
 747             }
 748 
 749             /**
 750              * Return true if there are no registered LiveRef instances for
 751              * this entry still reachable in this VM.
 752              *
 753              * This method must ONLY be invoked while synchronized on this
 754              * RefEntry's EndpointEntry.
 755              */
 756             public boolean isRefSetEmpty() {
 757                 assert Thread.holdsLock(EndpointEntry.this);
 758                 return refSet.size() == 0;
 759             }
 760 
 761             /**
 762              * Record that a dirty call that explicitly contained this
 763              * entry's ref has failed.
 764              *
 765              * This method must ONLY be invoked while synchronized on this
 766              * RefEntry's EndpointEntry.
 767              */
 768             public void markDirtyFailed() {
 769                 assert Thread.holdsLock(EndpointEntry.this);
 770                 dirtyFailed = true;
 771             }
 772 
 773             /**
 774              * Return true if a dirty call that explicitly contained this
 775              * entry's ref has failed (and therefore a clean call for this
 776              * ref needs to be marked "strong").
 777              *
 778              * This method must ONLY be invoked while synchronized on this
 779              * RefEntry's EndpointEntry.
 780              */
 781             public boolean hasDirtyFailed() {
 782                 assert Thread.holdsLock(EndpointEntry.this);
 783                 return dirtyFailed;
 784             }
 785 
 786             /**
 787              * PhantomLiveRef is a PhantomReference to a LiveRef instance,
 788              * used to detect when the LiveRef becomes permanently
 789              * unreachable in this VM.
 790              */
 791             private class PhantomLiveRef extends PhantomReference<LiveRef> {
 792 
 793                 public PhantomLiveRef(LiveRef ref) {
 794                     super(ref, EndpointEntry.this.refQueue);
 795                 }
 796 
 797                 public RefEntry getRefEntry() {
 798                     return RefEntry.this;
 799                 }
 800             }
 801         }
 802     }
 803 }