1 /*
   2  * Copyright (c) 1996, 2012, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package sun.rmi.transport;
  26 
  27 import java.lang.ref.PhantomReference;
  28 import java.lang.ref.ReferenceQueue;
  29 import java.security.AccessController;
  30 import java.security.PrivilegedAction;
  31 import java.util.HashMap;
  32 import java.util.HashSet;
  33 import java.util.Iterator;
  34 import java.util.List;
  35 import java.util.Map;
  36 import java.util.Set;
  37 import java.rmi.ConnectException;
  38 import java.rmi.RemoteException;
  39 import java.rmi.dgc.DGC;
  40 import java.rmi.dgc.Lease;
  41 import java.rmi.dgc.VMID;
  42 import java.rmi.server.ObjID;
  43 import sun.misc.GC;
  44 import sun.rmi.runtime.NewThreadAction;
  45 import sun.rmi.server.UnicastRef;
  46 import sun.rmi.server.Util;
  47 import sun.security.action.GetLongAction;
  48 
  49 /**
  50  * DGCClient implements the client-side of the RMI distributed garbage
  51  * collection system.
  52  *
  53  * The external interface to DGCClient is the "registerRefs" method.
  54  * When a LiveRef to a remote object enters the VM, it needs to be
  55  * registered with the DGCClient to participate in distributed garbage
  56  * collection.
  57  *
  58  * When the first LiveRef to a particular remote object is registered,
  59  * a "dirty" call is made to the server-side distributed garbage
  60  * collector for the remote object, which returns a lease guaranteeing
  61  * that the server-side DGC will not collect the remote object for a
  62  * certain period of time.  While LiveRef instances to remote objects
  63  * on a particular server exist, the DGCClient periodically sends more
  64  * "dirty" calls to renew its lease.
  65  *
  66  * The DGCClient tracks the local reachability of registered LiveRef
  67  * instances (using phantom references).  When the LiveRef instance
  68  * for a particular remote object becomes garbage collected locally,
  69  * a "clean" call is made to the server-side distributed garbage
  70  * collector, indicating that the server no longer needs to keep the
  71  * remote object alive for this client.
  72  *
  73  * @see java.rmi.dgc.DGC, sun.rmi.transport.DGCImpl
  74  *
  75  * @author  Ann Wollrath
  76  * @author  Peter Jones
  77  */
  78 final class DGCClient {
  79 
  80     /** next sequence number for DGC calls (access synchronized on class) */
  81     private static long nextSequenceNum = Long.MIN_VALUE;
  82 
  83     /** unique identifier for this VM as a client of DGC */
  84     private static VMID vmid = new VMID();
  85 
  86     /** lease duration to request (usually ignored by server) */
  87     private static final long leaseValue =              // default 10 minutes
  88         AccessController.doPrivileged(
  89             new GetLongAction("java.rmi.dgc.leaseValue",
  90                               600000)).longValue();
  91 
  92     /** maximum interval between retries of failed clean calls */
  93     private static final long cleanInterval =           // default 3 minutes
  94         AccessController.doPrivileged(
  95             new GetLongAction("sun.rmi.dgc.cleanInterval",
  96                               180000)).longValue();
  97 
  98     /** maximum interval between complete garbage collections of local heap */
  99     private static final long gcInterval =              // default 1 hour
 100         AccessController.doPrivileged(
 101             new GetLongAction("sun.rmi.dgc.client.gcInterval",
 102                               3600000)).longValue();
 103 
 104     /** minimum retry count for dirty calls that fail */
 105     private static final int dirtyFailureRetries = 5;
 106 
 107     /** retry count for clean calls that fail with ConnectException */
 108     private static final int cleanFailureRetries = 5;
 109 
 110     /** constant empty ObjID array for lease renewal optimization */
 111     private static final ObjID[] emptyObjIDArray = new ObjID[0];
 112 
 113     /** ObjID for server-side DGC object */
 114     private static final ObjID dgcID = new ObjID(ObjID.DGC_ID);
 115 
 116     /*
 117      * Disallow anyone from creating one of these.
 118      */
 119     private DGCClient() {}
 120 
 121     /**
 122      * Register the LiveRef instances in the supplied list to participate
 123      * in distributed garbage collection.
 124      *
 125      * All of the LiveRefs in the list must be for remote objects at the
 126      * given endpoint.
 127      */
 128     static void registerRefs(Endpoint ep, List<LiveRef> refs) {
 129         /*
 130          * Look up the given endpoint and register the refs with it.
 131          * The retrieved entry may get removed from the global endpoint
 132          * table before EndpointEntry.registerRefs() is able to acquire
 133          * its lock; in this event, it returns false, and we loop and
 134          * try again.
 135          */
 136         EndpointEntry epEntry;
 137         do {
 138             epEntry = EndpointEntry.lookup(ep);
 139         } while (!epEntry.registerRefs(refs));
 140     }
 141 
 142     /**
 143      * Get the next sequence number to be used for a dirty or clean
 144      * operation from this VM.  This method should only be called while
 145      * synchronized on the EndpointEntry whose data structures the
 146      * operation affects.
 147      */
 148     private static synchronized long getNextSequenceNum() {
 149         return nextSequenceNum++;
 150     }
 151 
 152     /**
 153      * Given the length of a lease and the time that it was granted,
 154      * compute the absolute time at which it should be renewed, giving
 155      * room for reasonable computational and communication delays.
 156      */
 157     private static long computeRenewTime(long grantTime, long duration) {
 158         /*
 159          * REMIND: This algorithm should be more sophisticated, waiting
 160          * a longer fraction of the lease duration for longer leases.
 161          */
 162         return grantTime + (duration / 2);
 163     }
 164 
 165     /**
 166      * EndpointEntry encapsulates the client-side DGC information specific
 167      * to a particular Endpoint.  Of most significance is the table that
 168      * maps LiveRef value to RefEntry objects and the renew/clean thread
 169      * that handles asynchronous client-side DGC operations.
 170      */
 171     private static class EndpointEntry {
 172 
 173         /** the endpoint that this entry is for */
 174         private Endpoint endpoint;
 175         /** synthesized reference to the remote server-side DGC */
 176         private DGC dgc;
 177 
 178         /** table of refs held for endpoint: maps LiveRef to RefEntry */
 179         private Map<LiveRef, RefEntry> refTable = new HashMap<>(5);
 180         /** set of RefEntry instances from last (failed) dirty call */
 181         private Set<RefEntry> invalidRefs = new HashSet<>(5);
 182 
 183         /** true if this entry has been removed from the global table */
 184         private boolean removed = false;
 185 
 186         /** absolute time to renew current lease to this endpoint */
 187         private long renewTime = Long.MAX_VALUE;
 188         /** absolute time current lease to this endpoint will expire */
 189         private long expirationTime = Long.MIN_VALUE;
 190         /** count of recent dirty calls that have failed */
 191         private int dirtyFailures = 0;
 192         /** absolute time of first recent failed dirty call */
 193         private long dirtyFailureStartTime;
 194         /** (average) elapsed time for recent failed dirty calls */
 195         private long dirtyFailureDuration;
 196 
 197         /** renew/clean thread for handling lease renewals and clean calls */
 198         private Thread renewCleanThread;
 199         /** true if renew/clean thread may be interrupted */
 200         private boolean interruptible = false;
 201 
 202         /** reference queue for phantom references */
 203         private ReferenceQueue<LiveRef> refQueue = new ReferenceQueue<>();
 204         /** set of clean calls that need to be made */
 205         private Set<CleanRequest> pendingCleans = new HashSet<>(5);
 206 
 207         /** global endpoint table: maps Endpoint to EndpointEntry */
 208         private static Map<Endpoint,EndpointEntry> endpointTable = new HashMap<>(5);
 209         /** handle for GC latency request (for future cancellation) */
 210         private static GC.LatencyRequest gcLatencyRequest = null;
 211 
 212         /**
 213          * Look up the EndpointEntry for the given Endpoint.  An entry is
 214          * created if one does not already exist.
 215          */
 216         public static EndpointEntry lookup(Endpoint ep) {
 217             synchronized (endpointTable) {
 218                 EndpointEntry entry = endpointTable.get(ep);
 219                 if (entry == null) {
 220                     entry = new EndpointEntry(ep);
 221                     endpointTable.put(ep, entry);
 222                     /*
 223                      * While we are tracking live remote references registered
 224                      * in this VM, request a maximum latency for inspecting the
 225                      * entire heap from the local garbage collector, to place
 226                      * an upper bound on the time to discover remote references
 227                      * that have become unreachable (see bugid 4171278).
 228                      */
 229                     if (gcLatencyRequest == null) {
 230                         gcLatencyRequest = GC.requestLatency(gcInterval);
 231                     }
 232                 }
 233                 return entry;
 234             }
 235         }
 236 
 237         private EndpointEntry(final Endpoint endpoint) {
 238             this.endpoint = endpoint;
 239             try {
 240                 LiveRef dgcRef = new LiveRef(dgcID, endpoint, false);
 241                 dgc = (DGC) Util.createProxy(DGCImpl.class,
 242                                              new UnicastRef(dgcRef), true);
 243             } catch (RemoteException e) {
 244                 throw new Error("internal error creating DGC stub");
 245             }
 246             renewCleanThread =  AccessController.doPrivileged(
 247                 new NewThreadAction(new RenewCleanThread(),
 248                                     "RenewClean-" + endpoint, true));
 249             renewCleanThread.start();
 250         }
 251 
 252         /**
 253          * Register the LiveRef instances in the supplied list to participate
 254          * in distributed garbage collection.
 255          *
 256          * This method returns false if this entry was removed from the
 257          * global endpoint table (because it was empty) before these refs
 258          * could be registered.  In that case, a new EndpointEntry needs
 259          * to be looked up.
 260          *
 261          * This method must NOT be called while synchronized on this entry.
 262          */
 263         public boolean registerRefs(List<LiveRef> refs) {
 264             assert !Thread.holdsLock(this);
 265 
 266             Set<RefEntry> refsToDirty = null;     // entries for refs needing dirty
 267             long sequenceNum;           // sequence number for dirty call
 268 
 269             synchronized (this) {
 270                 if (removed) {
 271                     return false;
 272                 }
 273 
 274                 Iterator<LiveRef> iter = refs.iterator();
 275                 while (iter.hasNext()) {
 276                     LiveRef ref = iter.next();
 277                     assert ref.getEndpoint().equals(endpoint);
 278 
 279                     RefEntry refEntry = refTable.get(ref);
 280                     if (refEntry == null) {
 281                         LiveRef refClone = (LiveRef) ref.clone();
 282                         refEntry = new RefEntry(refClone);
 283                         refTable.put(refClone, refEntry);
 284                         if (refsToDirty == null) {
 285                             refsToDirty = new HashSet<>(5);
 286                         }
 287                         refsToDirty.add(refEntry);
 288                     }
 289 
 290                     refEntry.addInstanceToRefSet(ref);
 291                 }
 292 
 293                 if (refsToDirty == null) {
 294                     return true;
 295                 }
 296 
 297                 refsToDirty.addAll(invalidRefs);
 298                 invalidRefs.clear();
 299 
 300                 sequenceNum = getNextSequenceNum();
 301             }
 302 
 303             makeDirtyCall(refsToDirty, sequenceNum);
 304             return true;
 305         }
 306 
 307         /**
 308          * Remove the given RefEntry from the ref table.  If that makes
 309          * the ref table empty, remove this entry from the global endpoint
 310          * table.
 311          *
 312          * This method must ONLY be called while synchronized on this entry.
 313          */
 314         private void removeRefEntry(RefEntry refEntry) {
 315             assert Thread.holdsLock(this);
 316             assert !removed;
 317             assert refTable.containsKey(refEntry.getRef());
 318 
 319             refTable.remove(refEntry.getRef());
 320             invalidRefs.remove(refEntry);
 321             if (refTable.isEmpty()) {
 322                 synchronized (endpointTable) {
 323                     endpointTable.remove(endpoint);
 324                     Transport transport = endpoint.getOutboundTransport();
 325                     transport.free(endpoint);
 326                     /*
 327                      * If there are no longer any live remote references
 328                      * registered, we are no longer concerned with the
 329                      * latency of local garbage collection here.
 330                      */
 331                     if (endpointTable.isEmpty()) {
 332                         assert gcLatencyRequest != null;
 333                         gcLatencyRequest.cancel();
 334                         gcLatencyRequest = null;
 335                     }
 336                     removed = true;
 337                 }
 338             }
 339         }
 340 
 341         /**
 342          * Make a DGC dirty call to this entry's endpoint, for the ObjIDs
 343          * corresponding to the given set of refs and with the given
 344          * sequence number.
 345          *
 346          * This method must NOT be called while synchronized on this entry.
 347          */
 348         private void makeDirtyCall(Set<RefEntry> refEntries, long sequenceNum) {
 349             assert !Thread.holdsLock(this);
 350 
 351             ObjID[] ids;
 352             if (refEntries != null) {
 353                 ids = createObjIDArray(refEntries);
 354             } else {
 355                 ids = emptyObjIDArray;
 356             }
 357 
 358             long startTime = System.currentTimeMillis();
 359             try {
 360                 Lease lease =
 361                     dgc.dirty(ids, sequenceNum, new Lease(vmid, leaseValue));
 362                 long duration = lease.getValue();
 363 
 364                 long newRenewTime = computeRenewTime(startTime, duration);
 365                 long newExpirationTime = startTime + duration;
 366 
 367                 synchronized (this) {
 368                     dirtyFailures = 0;
 369                     setRenewTime(newRenewTime);
 370                     expirationTime = newExpirationTime;
 371                 }
 372 
 373             } catch (Exception e) {
 374                 long endTime = System.currentTimeMillis();
 375 
 376                 synchronized (this) {
 377                     dirtyFailures++;
 378 
 379                     if (dirtyFailures == 1) {
 380                         /*
 381                          * If this was the first recent failed dirty call,
 382                          * reschedule another one immediately, in case there
 383                          * was just a transient network problem, and remember
 384                          * the start time and duration of this attempt for
 385                          * future calculations of the delays between retries.
 386                          */
 387                         dirtyFailureStartTime = startTime;
 388                         dirtyFailureDuration = endTime - startTime;
 389                         setRenewTime(endTime);
 390                     } else {
 391                         /*
 392                          * For each successive failed dirty call, wait for a
 393                          * (binary) exponentially increasing delay before
 394                          * retrying, to avoid network congestion.
 395                          */
 396                         int n = dirtyFailures - 2;
 397                         if (n == 0) {
 398                             /*
 399                              * Calculate the initial retry delay from the
 400                              * average time elapsed for each of the first
 401                              * two failed dirty calls.  The result must be
 402                              * at least 1000ms, to prevent a tight loop.
 403                              */
 404                             dirtyFailureDuration =
 405                                 Math.max((dirtyFailureDuration +
 406                                           (endTime - startTime)) >> 1, 1000);
 407                         }
 408                         long newRenewTime =
 409                             endTime + (dirtyFailureDuration << n);
 410 
 411                         /*
 412                          * Continue if the last known held lease has not
 413                          * expired, or else at least a fixed number of times,
 414                          * or at least until we've tried for a fixed amount
 415                          * of time (the default lease value we request).
 416                          */
 417                         if (newRenewTime < expirationTime ||
 418                             dirtyFailures < dirtyFailureRetries ||
 419                             newRenewTime < dirtyFailureStartTime + leaseValue)
 420                         {
 421                             setRenewTime(newRenewTime);
 422                         } else {
 423                             /*
 424                              * Give up: postpone lease renewals until next
 425                              * ref is registered for this endpoint.
 426                              */
 427                             setRenewTime(Long.MAX_VALUE);
 428                         }
 429                     }
 430 
 431                     if (refEntries != null) {
 432                         /*
 433                          * Add all of these refs to the set of refs for this
 434                          * endpoint that may be invalid (this VM may not be in
 435                          * the server's referenced set), so that we will
 436                          * attempt to explicitly dirty them again in the
 437                          * future.
 438                          */
 439                         invalidRefs.addAll(refEntries);
 440 
 441                         /*
 442                          * Record that a dirty call has failed for all of these
 443                          * refs, so that clean calls for them in the future
 444                          * will be strong.
 445                          */
 446                         Iterator<RefEntry> iter = refEntries.iterator();
 447                         while (iter.hasNext()) {
 448                             RefEntry refEntry = iter.next();
 449                             refEntry.markDirtyFailed();
 450                         }
 451                     }
 452 
 453                     /*
 454                      * If the last known held lease will have expired before
 455                      * the next renewal, all refs might be invalid.
 456                      */
 457                     if (renewTime >= expirationTime) {
 458                         invalidRefs.addAll(refTable.values());
 459                     }
 460                 }
 461             }
 462         }
 463 
 464         /**
 465          * Set the absolute time at which the lease for this entry should
 466          * be renewed.
 467          *
 468          * This method must ONLY be called while synchronized on this entry.
 469          */
 470         private void setRenewTime(long newRenewTime) {
 471             assert Thread.holdsLock(this);
 472 
 473             if (newRenewTime < renewTime) {
 474                 renewTime = newRenewTime;
 475                 if (interruptible) {
 476                     AccessController.doPrivileged(
 477                         new PrivilegedAction<Void>() {
 478                             public Void run() {
 479                             renewCleanThread.interrupt();
 480                             return null;
 481                         }
 482                     });
 483                 }
 484             } else {
 485                 renewTime = newRenewTime;
 486             }
 487         }
 488 
 489         /**
 490          * RenewCleanThread handles the asynchronous client-side DGC activity
 491          * for this entry: renewing the leases and making clean calls.
 492          */
 493         private class RenewCleanThread implements Runnable {
 494 
 495             public void run() {
 496                 do {
 497                     long timeToWait;
 498                     RefEntry.PhantomLiveRef phantom = null;
 499                     boolean needRenewal = false;
 500                     Set<RefEntry> refsToDirty = null;
 501                     long sequenceNum = Long.MIN_VALUE;
 502 
 503                     synchronized (EndpointEntry.this) {
 504                         /*
 505                          * Calculate time to block (waiting for phantom
 506                          * reference notifications).  It is the time until the
 507                          * lease renewal should be done, bounded on the low
 508                          * end by 1 ms so that the reference queue will always
 509                          * get processed, and if there are pending clean
 510                          * requests (remaining because some clean calls
 511                          * failed), bounded on the high end by the maximum
 512                          * clean call retry interval.
 513                          */
 514                         long timeUntilRenew =
 515                             renewTime - System.currentTimeMillis();
 516                         timeToWait = Math.max(timeUntilRenew, 1);
 517                         if (!pendingCleans.isEmpty()) {
 518                             timeToWait = Math.min(timeToWait, cleanInterval);
 519                         }
 520 
 521                         /*
 522                          * Set flag indicating that it is OK to interrupt this
 523                          * thread now, such as if a earlier lease renewal time
 524                          * is set, because we are only going to be blocking
 525                          * and can deal with interrupts.
 526                          */
 527                         interruptible = true;
 528                     }
 529 
 530                     try {
 531                         /*
 532                          * Wait for the duration calculated above for any of
 533                          * our phantom references to be enqueued.
 534                          */
 535                         phantom = (RefEntry.PhantomLiveRef)
 536                             refQueue.remove(timeToWait);
 537                     } catch (InterruptedException e) {
 538                     }
 539 
 540                     synchronized (EndpointEntry.this) {
 541                         /*
 542                          * Set flag indicating that it is NOT OK to interrupt
 543                          * this thread now, because we may be undertaking I/O
 544                          * operations that should not be interrupted (and we
 545                          * will not be blocking arbitrarily).
 546                          */
 547                         interruptible = false;
 548                         Thread.interrupted();   // clear interrupted state
 549 
 550                         /*
 551                          * If there was a phantom reference enqueued, process
 552                          * it and all the rest on the queue, generating
 553                          * clean requests as necessary.
 554                          */
 555                         if (phantom != null) {
 556                             processPhantomRefs(phantom);
 557                         }
 558 
 559                         /*
 560                          * Check if it is time to renew this entry's lease.
 561                          */
 562                         long currentTime = System.currentTimeMillis();
 563                         if (currentTime > renewTime) {
 564                             needRenewal = true;
 565                             if (!invalidRefs.isEmpty()) {
 566                                 refsToDirty = invalidRefs;
 567                                 invalidRefs = new HashSet<>(5);
 568                             }
 569                             sequenceNum = getNextSequenceNum();
 570                         }
 571                     }
 572 
 573                     if (needRenewal) {
 574                         makeDirtyCall(refsToDirty, sequenceNum);
 575                     }
 576 
 577                     if (!pendingCleans.isEmpty()) {
 578                         makeCleanCalls();
 579                     }
 580                 } while (!removed || !pendingCleans.isEmpty());
 581             }
 582         }
 583 
 584         /**
 585          * Process the notification of the given phantom reference and any
 586          * others that are on this entry's reference queue.  Each phantom
 587          * reference is removed from its RefEntry's ref set.  All ref
 588          * entries that have no more registered instances are collected
 589          * into up to two batched clean call requests: one for refs
 590          * requiring a "strong" clean call, and one for the rest.
 591          *
 592          * This method must ONLY be called while synchronized on this entry.
 593          */
 594         private void processPhantomRefs(RefEntry.PhantomLiveRef phantom) {
 595             assert Thread.holdsLock(this);
 596 
 597             Set<RefEntry> strongCleans = null;
 598             Set<RefEntry> normalCleans = null;
 599 
 600             do {
 601                 RefEntry refEntry = phantom.getRefEntry();
 602                 refEntry.removeInstanceFromRefSet(phantom);
 603                 if (refEntry.isRefSetEmpty()) {
 604                     if (refEntry.hasDirtyFailed()) {
 605                         if (strongCleans == null) {
 606                             strongCleans = new HashSet<>(5);
 607                         }
 608                         strongCleans.add(refEntry);
 609                     } else {
 610                         if (normalCleans == null) {
 611                             normalCleans = new HashSet<>(5);
 612                         }
 613                         normalCleans.add(refEntry);
 614                     }
 615                     removeRefEntry(refEntry);
 616                 }
 617             } while ((phantom =
 618                 (RefEntry.PhantomLiveRef) refQueue.poll()) != null);
 619 
 620             if (strongCleans != null) {
 621                 pendingCleans.add(
 622                     new CleanRequest(createObjIDArray(strongCleans),
 623                                      getNextSequenceNum(), true));
 624             }
 625             if (normalCleans != null) {
 626                 pendingCleans.add(
 627                     new CleanRequest(createObjIDArray(normalCleans),
 628                                      getNextSequenceNum(), false));
 629             }
 630         }
 631 
 632         /**
 633          * CleanRequest holds the data for the parameters of a clean call
 634          * that needs to be made.
 635          */
 636         private static class CleanRequest {
 637 
 638             final ObjID[] objIDs;
 639             final long sequenceNum;
 640             final boolean strong;
 641 
 642             /** how many times this request has failed */
 643             int failures = 0;
 644 
 645             CleanRequest(ObjID[] objIDs, long sequenceNum, boolean strong) {
 646                 this.objIDs = objIDs;
 647                 this.sequenceNum = sequenceNum;
 648                 this.strong = strong;
 649             }
 650         }
 651 
 652         /**
 653          * Make all of the clean calls described by the clean requests in
 654          * this entry's set of "pending cleans".  Clean requests for clean
 655          * calls that succeed are removed from the "pending cleans" set.
 656          *
 657          * This method must NOT be called while synchronized on this entry.
 658          */
 659         private void makeCleanCalls() {
 660             assert !Thread.holdsLock(this);
 661 
 662             Iterator<CleanRequest> iter = pendingCleans.iterator();
 663             while (iter.hasNext()) {
 664                 CleanRequest request = iter.next();
 665                 try {
 666                     dgc.clean(request.objIDs, request.sequenceNum, vmid,
 667                               request.strong);
 668                     iter.remove();
 669                 } catch (Exception e) {
 670                     /*
 671                      * Many types of exceptions here could have been
 672                      * caused by a transient failure, so try again a
 673                      * few times, but not forever.
 674                      */
 675                     if (++request.failures >= cleanFailureRetries) {
 676                         iter.remove();
 677                     }
 678                 }
 679             }
 680         }
 681 
 682         /**
 683          * Create an array of ObjIDs (needed for the DGC remote calls)
 684          * from the ids in the given set of refs.
 685          */
 686         private static ObjID[] createObjIDArray(Set<RefEntry> refEntries) {
 687             ObjID[] ids = new ObjID[refEntries.size()];
 688             Iterator<RefEntry> iter = refEntries.iterator();
 689             for (int i = 0; i < ids.length; i++) {
 690                 ids[i] = iter.next().getRef().getObjID();
 691             }
 692             return ids;
 693         }
 694 
 695         /**
 696          * RefEntry encapsulates the client-side DGC information specific
 697          * to a particular LiveRef value.  In particular, it contains a
 698          * set of phantom references to all of the instances of the LiveRef
 699          * value registered in the system (but not garbage collected
 700          * locally).
 701          */
 702         private class RefEntry {
 703 
 704             /** LiveRef value for this entry (not a registered instance) */
 705             private LiveRef ref;
 706             /** set of phantom references to registered instances */
 707             private Set<PhantomLiveRef> refSet = new HashSet<>(5);
 708             /** true if a dirty call containing this ref has failed */
 709             private boolean dirtyFailed = false;
 710 
 711             public RefEntry(LiveRef ref) {
 712                 this.ref = ref;
 713             }
 714 
 715             /**
 716              * Return the LiveRef value for this entry (not a registered
 717              * instance).
 718              */
 719             public LiveRef getRef() {
 720                 return ref;
 721             }
 722 
 723             /**
 724              * Add a LiveRef to the set of registered instances for this entry.
 725              *
 726              * This method must ONLY be invoked while synchronized on this
 727              * RefEntry's EndpointEntry.
 728              */
 729             public void addInstanceToRefSet(LiveRef ref) {
 730                 assert Thread.holdsLock(EndpointEntry.this);
 731                 assert ref.equals(this.ref);
 732 
 733                 /*
 734                  * Only keep a phantom reference to the registered instance,
 735                  * so that it can be garbage collected normally (and we can be
 736                  * notified when that happens).
 737                  */
 738                 refSet.add(new PhantomLiveRef(ref));
 739             }
 740 
 741             /**
 742              * Remove a PhantomLiveRef from the set of registered instances.
 743              *
 744              * This method must ONLY be invoked while synchronized on this
 745              * RefEntry's EndpointEntry.
 746              */
 747             public void removeInstanceFromRefSet(PhantomLiveRef phantom) {
 748                 assert Thread.holdsLock(EndpointEntry.this);
 749                 assert refSet.contains(phantom);
 750                 refSet.remove(phantom);
 751             }
 752 
 753             /**
 754              * Return true if there are no registered LiveRef instances for
 755              * this entry still reachable in this VM.
 756              *
 757              * This method must ONLY be invoked while synchronized on this
 758              * RefEntry's EndpointEntry.
 759              */
 760             public boolean isRefSetEmpty() {
 761                 assert Thread.holdsLock(EndpointEntry.this);
 762                 return refSet.size() == 0;
 763             }
 764 
 765             /**
 766              * Record that a dirty call that explicitly contained this
 767              * entry's ref has failed.
 768              *
 769              * This method must ONLY be invoked while synchronized on this
 770              * RefEntry's EndpointEntry.
 771              */
 772             public void markDirtyFailed() {
 773                 assert Thread.holdsLock(EndpointEntry.this);
 774                 dirtyFailed = true;
 775             }
 776 
 777             /**
 778              * Return true if a dirty call that explicitly contained this
 779              * entry's ref has failed (and therefore a clean call for this
 780              * ref needs to be marked "strong").
 781              *
 782              * This method must ONLY be invoked while synchronized on this
 783              * RefEntry's EndpointEntry.
 784              */
 785             public boolean hasDirtyFailed() {
 786                 assert Thread.holdsLock(EndpointEntry.this);
 787                 return dirtyFailed;
 788             }
 789 
 790             /**
 791              * PhantomLiveRef is a PhantomReference to a LiveRef instance,
 792              * used to detect when the LiveRef becomes permanently
 793              * unreachable in this VM.
 794              */
 795             private class PhantomLiveRef extends PhantomReference<LiveRef> {
 796 
 797                 public PhantomLiveRef(LiveRef ref) {
 798                     super(ref, EndpointEntry.this.refQueue);
 799                 }
 800 
 801                 public RefEntry getRefEntry() {
 802                     return RefEntry.this;
 803                 }
 804             }
 805         }
 806     }
 807 }