1 /*
   2  * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 /*
  25  * @test
  26  * @bug 4915825 4921009 4934965 4977469 8019584
  27  * @summary Tests behavior when client or server gets object of unknown class
  28  * @author Eamonn McManus
  29  * @modules java.management.rmi
  30  * @run clean MissingClassTest SingleClassLoader
  31  * @run build MissingClassTest SingleClassLoader
  32  * @run main MissingClassTest
  33  * @key randomness
  34  */
  35 
  36 /*
  37   Tests that clients and servers react correctly when they receive
  38   objects of unknown classes.  This can happen easily due to version
  39   skew or missing jar files on one end or the other.  The default
  40   behaviour of causing a connection to die because of the resultant
  41   IOException is not acceptable!  We try sending attributes and invoke
  42   parameters to the server of classes it doesn't know, and we try
  43   sending attributes, exceptions and notifications to the client of
  44   classes it doesn't know.
  45 
  46   We also test objects that are of known class but not serializable.
  47   The test cases are similar.
  48  */
  49 
  50 import java.io.ByteArrayOutputStream;
  51 import java.io.IOException;
  52 import java.io.NotSerializableException;
  53 import java.io.ObjectOutputStream;
  54 import java.net.MalformedURLException;
  55 import java.util.HashMap;
  56 import java.util.Map;
  57 import java.util.Random;
  58 import java.util.Set;
  59 import javax.management.Attribute;
  60 import javax.management.MBeanServer;
  61 import javax.management.MBeanServerConnection;
  62 import javax.management.MBeanServerFactory;
  63 import javax.management.Notification;
  64 import javax.management.NotificationBroadcasterSupport;
  65 import javax.management.NotificationFilter;
  66 import javax.management.NotificationListener;
  67 import javax.management.ObjectName;
  68 import javax.management.remote.JMXConnectionNotification;
  69 import javax.management.remote.JMXConnector;
  70 import javax.management.remote.JMXConnectorFactory;
  71 import javax.management.remote.JMXConnectorServer;
  72 import javax.management.remote.JMXConnectorServerFactory;
  73 import javax.management.remote.JMXServiceURL;
  74 import javax.management.remote.rmi.RMIConnectorServer;
  75 
  76 public class MissingClassTest {
  77     private static final int NNOTIFS = 50;
  78 
  79     private static ClassLoader clientLoader, serverLoader;
  80     private static Object serverUnknown;
  81     private static Exception clientUnknown;
  82     private static ObjectName on;
  83     private static final Object[] NO_OBJECTS = new Object[0];
  84     private static final String[] NO_STRINGS = new String[0];
  85 
  86     private static final Object unserializableObject = Thread.currentThread();
  87 
  88     private static boolean isInstance(Object o, String cn) {
  89         try {
  90             Class<?> c = Class.forName(cn);
  91             return c.isInstance(o);
  92         } catch (ClassNotFoundException x) {
  93             return false;
  94         }
  95     }
  96 
  97     public static void main(String[] args) throws Exception {
  98         System.out.println("Test that the client or server end of a " +
  99                            "connection does not fail if sent an object " +
 100                            "it cannot deserialize");
 101 
 102         on = new ObjectName("test:type=Test");
 103 
 104         ClassLoader testLoader = MissingClassTest.class.getClassLoader();
 105         clientLoader =
 106             new SingleClassLoader("$ServerUnknown$", HashMap.class,
 107                                   testLoader);
 108         serverLoader =
 109             new SingleClassLoader("$ClientUnknown$", Exception.class,
 110                                   testLoader);
 111         serverUnknown =
 112             clientLoader.loadClass("$ServerUnknown$").newInstance();
 113         clientUnknown = (Exception)
 114             serverLoader.loadClass("$ClientUnknown$").newInstance();
 115 
 116         final String[] protos = {"rmi", /*"iiop",*/ "jmxmp"};
 117         boolean ok = true;
 118         for (int i = 0; i < protos.length; i++) {
 119             try {
 120                 ok &= test(protos[i]);
 121             } catch (Exception e) {
 122                 System.out.println("TEST FAILED WITH EXCEPTION:");
 123                 e.printStackTrace(System.out);
 124                 ok = false;
 125             }
 126         }
 127 
 128         if (ok)
 129             System.out.println("Test passed");
 130         else {
 131             throw new RuntimeException("TEST FAILED");
 132         }
 133     }
 134 
 135     private static boolean test(String proto) throws Exception {
 136         System.out.println("Testing for proto " + proto);
 137 
 138         boolean ok = true;
 139 
 140         MBeanServer mbs = MBeanServerFactory.newMBeanServer();
 141         mbs.createMBean(Test.class.getName(), on);
 142 
 143         JMXConnectorServer cs;
 144         JMXServiceURL url = new JMXServiceURL(proto, null, 0);
 145         Map<String,Object> serverMap = new HashMap<>();
 146         serverMap.put(JMXConnectorServerFactory.DEFAULT_CLASS_LOADER,
 147                       serverLoader);
 148 
 149         // make sure no auto-close at server side
 150         serverMap.put("jmx.remote.x.server.connection.timeout", "888888888");
 151 
 152         try {
 153             cs = JMXConnectorServerFactory.newJMXConnectorServer(url,
 154                                                                  serverMap,
 155                                                                  mbs);
 156         } catch (MalformedURLException e) {
 157             System.out.println("System does not recognize URL: " + url +
 158                                "; ignoring");
 159             return true;
 160         }
 161         cs.start();
 162         JMXServiceURL addr = cs.getAddress();
 163         Map<String,Object> clientMap = new HashMap<>();
 164         clientMap.put(JMXConnectorFactory.DEFAULT_CLASS_LOADER,
 165                       clientLoader);
 166 
 167         System.out.println("Connecting for client-unknown test");
 168 
 169         JMXConnector client = JMXConnectorFactory.connect(addr, clientMap);
 170 
 171         // add a listener to verify no failed notif
 172         CNListener cnListener = new CNListener();
 173         client.addConnectionNotificationListener(cnListener, null, null);
 174 
 175         MBeanServerConnection mbsc = client.getMBeanServerConnection();
 176 
 177         System.out.println("Getting attribute with class unknown to client");
 178         try {
 179             Object result = mbsc.getAttribute(on, "ClientUnknown");
 180             System.out.println("TEST FAILS: getAttribute for class " +
 181                                "unknown to client should fail, returned: " +
 182                                result);
 183             ok = false;
 184         } catch (IOException e) {
 185             Throwable cause = e.getCause();
 186             if (isInstance(cause, "org.omg.CORBA.MARSHAL"))  // see CR 4935098
 187                 cause = cause.getCause();
 188             if (cause instanceof ClassNotFoundException) {
 189                 System.out.println("Success: got an IOException wrapping " +
 190                                    "a ClassNotFoundException");
 191             } else {
 192                 System.out.println("TEST FAILS: Caught IOException (" + e +
 193                                    ") but cause should be " +
 194                                    "ClassNotFoundException: " + cause);
 195                 ok = false;
 196             }
 197         }
 198 
 199         System.out.println("Doing queryNames to ensure connection alive");
 200         Set<ObjectName> names = mbsc.queryNames(null, null);
 201         System.out.println("queryNames returned " + names);
 202 
 203         System.out.println("Provoke exception of unknown class");
 204         try {
 205             mbsc.invoke(on, "throwClientUnknown", NO_OBJECTS, NO_STRINGS);
 206             System.out.println("TEST FAILS: did not get exception");
 207             ok = false;
 208         } catch (IOException e) {
 209             Throwable wrapped = e.getCause();
 210             if (isInstance(wrapped, "org.omg.CORBA.MARSHAL"))  // see CR 4935098
 211                 wrapped = wrapped.getCause();
 212             if (wrapped instanceof ClassNotFoundException) {
 213                 System.out.println("Success: got an IOException wrapping " +
 214                                    "a ClassNotFoundException: " +
 215                                    wrapped);
 216             } else {
 217                 System.out.println("TEST FAILS: Got IOException but cause " +
 218                                    "should be ClassNotFoundException: ");
 219                 if (wrapped == null)
 220                     System.out.println("(null)");
 221                 else
 222                     wrapped.printStackTrace(System.out);
 223                 ok = false;
 224             }
 225         } catch (Exception e) {
 226             System.out.println("TEST FAILS: Got wrong exception: " +
 227                                "should be IOException with cause " +
 228                                "ClassNotFoundException:");
 229             e.printStackTrace(System.out);
 230             ok = false;
 231         }
 232 
 233         System.out.println("Doing queryNames to ensure connection alive");
 234         names = mbsc.queryNames(null, null);
 235         System.out.println("queryNames returned " + names);
 236 
 237         ok &= notifyTest(client, mbsc);
 238 
 239         System.out.println("Doing queryNames to ensure connection alive");
 240         names = mbsc.queryNames(null, null);
 241         System.out.println("queryNames returned " + names);
 242 
 243         for (int i = 0; i < 2; i++) {
 244             boolean setAttribute = (i == 0); // else invoke
 245             String what = setAttribute ? "setAttribute" : "invoke";
 246             System.out.println("Trying " + what +
 247                                " with class unknown to server");
 248             try {
 249                 if (setAttribute) {
 250                     mbsc.setAttribute(on, new Attribute("ServerUnknown",
 251                                                         serverUnknown));
 252                 } else {
 253                     mbsc.invoke(on, "useServerUnknown",
 254                                 new Object[] {serverUnknown},
 255                                 new String[] {"java.lang.Object"});
 256                 }
 257                 System.out.println("TEST FAILS: " + what + " with " +
 258                                    "class unknown to server should fail " +
 259                                    "but did not");
 260                 ok = false;
 261             } catch (IOException e) {
 262                 Throwable cause = e.getCause();
 263                 if (isInstance(cause, "org.omg.CORBA.MARSHAL"))  // see CR 4935098
 264                     cause = cause.getCause();
 265                 if (cause instanceof ClassNotFoundException) {
 266                     System.out.println("Success: got an IOException " +
 267                                        "wrapping a ClassNotFoundException");
 268                 } else {
 269                     System.out.println("TEST FAILS: Caught IOException (" + e +
 270                                        ") but cause should be " +
 271                                        "ClassNotFoundException: " + cause);
 272                     e.printStackTrace(System.out); // XXX
 273                     ok = false;
 274                 }
 275             }
 276         }
 277 
 278         System.out.println("Doing queryNames to ensure connection alive");
 279         names = mbsc.queryNames(null, null);
 280         System.out.println("queryNames returned " + names);
 281 
 282         System.out.println("Trying to get unserializable attribute");
 283         try {
 284             mbsc.getAttribute(on, "Unserializable");
 285             System.out.println("TEST FAILS: get unserializable worked " +
 286                                "but should not");
 287             ok = false;
 288         } catch (IOException e) {
 289             System.out.println("Success: got an IOException: " + e +
 290                                " (cause: " + e.getCause() + ")");
 291         }
 292 
 293         System.out.println("Doing queryNames to ensure connection alive");
 294         names = mbsc.queryNames(null, null);
 295         System.out.println("queryNames returned " + names);
 296 
 297         System.out.println("Trying to set unserializable attribute");
 298         try {
 299             Attribute attr =
 300                 new Attribute("Unserializable", unserializableObject);
 301             mbsc.setAttribute(on, attr);
 302             System.out.println("TEST FAILS: set unserializable worked " +
 303                                "but should not");
 304             ok = false;
 305         } catch (IOException e) {
 306             System.out.println("Success: got an IOException: " + e +
 307                                " (cause: " + e.getCause() + ")");
 308         }
 309 
 310         System.out.println("Doing queryNames to ensure connection alive");
 311         names = mbsc.queryNames(null, null);
 312         System.out.println("queryNames returned " + names);
 313 
 314         System.out.println("Trying to throw unserializable exception");
 315         try {
 316             mbsc.invoke(on, "throwUnserializable", NO_OBJECTS, NO_STRINGS);
 317             System.out.println("TEST FAILS: throw unserializable worked " +
 318                                "but should not");
 319             ok = false;
 320         } catch (IOException e) {
 321             System.out.println("Success: got an IOException: " + e +
 322                                " (cause: " + e.getCause() + ")");
 323         }
 324 
 325         client.removeConnectionNotificationListener(cnListener);
 326         ok = ok && !cnListener.failed;
 327 
 328         client.close();
 329         cs.stop();
 330 
 331         if (ok)
 332             System.out.println("Test passed for protocol " + proto);
 333 
 334         System.out.println();
 335         return ok;
 336     }
 337 
 338     private static class TestListener implements NotificationListener {
 339         TestListener(LostListener ll) {
 340             this.lostListener = ll;
 341         }
 342 
 343         public void handleNotification(Notification n, Object h) {
 344             /* Connectors can handle unserializable notifications in
 345                one of two ways.  Either they can arrange for the
 346                client to get a NotSerializableException from its
 347                fetchNotifications call (RMI connector), or they can
 348                replace the unserializable notification by a
 349                JMXConnectionNotification.NOTIFS_LOST (JMXMP
 350                connector).  The former case is handled by code within
 351                the connector client which will end up sending a
 352                NOTIFS_LOST to our LostListener.  The logic here
 353                handles the latter case by converting it into the
 354                former.
 355              */
 356             if (n instanceof JMXConnectionNotification
 357                 && n.getType().equals(JMXConnectionNotification.NOTIFS_LOST)) {
 358                 lostListener.handleNotification(n, h);
 359                 return;
 360             }
 361 
 362             synchronized (result) {
 363                 if (!n.getType().equals("interesting")
 364                     || !n.getUserData().equals("known")) {
 365                     System.out.println("TestListener received strange notif: "
 366                                        + notificationString(n));
 367                     result.failed = true;
 368                     result.notifyAll();
 369                 } else {
 370                     result.knownCount++;
 371                     if (result.knownCount == NNOTIFS)
 372                         result.notifyAll();
 373                 }
 374             }
 375         }
 376 
 377         private LostListener lostListener;
 378     }
 379 
 380     private static class LostListener implements NotificationListener {
 381         public void handleNotification(Notification n, Object h) {
 382             synchronized (result) {
 383                 handle(n, h);
 384             }
 385         }
 386 
 387         private void handle(Notification n, Object h) {
 388             if (!(n instanceof JMXConnectionNotification)) {
 389                 System.out.println("LostListener received strange notif: " +
 390                                    notificationString(n));
 391                 result.failed = true;
 392                 result.notifyAll();
 393                 return;
 394             }
 395 
 396             JMXConnectionNotification jn = (JMXConnectionNotification) n;
 397             if (!jn.getType().equals(jn.NOTIFS_LOST)) {
 398                 System.out.println("Ignoring JMXConnectionNotification: " +
 399                                    notificationString(jn));
 400                 return;
 401             }
 402             final String msg = jn.getMessage();
 403             if ((!msg.startsWith("Dropped ")
 404                  || !msg.endsWith("classes were missing locally"))
 405                 && (!msg.startsWith("Not serializable: "))) {
 406                 System.out.println("Surprising NOTIFS_LOST getMessage: " +
 407                                    msg);
 408             }
 409             if (!(jn.getUserData() instanceof Long)) {
 410                 System.out.println("JMXConnectionNotification userData " +
 411                                    "not a Long: " + jn.getUserData());
 412                 result.failed = true;
 413             } else {
 414                 int lost = ((Long) jn.getUserData()).intValue();
 415                 result.lostCount += lost;
 416                 if (result.lostCount == NNOTIFS*2)
 417                     result.notifyAll();
 418             }
 419         }
 420     }
 421 
 422     private static class TestFilter implements NotificationFilter {
 423         public boolean isNotificationEnabled(Notification n) {
 424             return (n.getType().equals("interesting"));
 425         }
 426     }
 427 
 428     private static class Result {
 429         int knownCount, lostCount;
 430         boolean failed;
 431     }
 432     private static Result result;
 433 
 434     /* Send a bunch of notifications to exercise the logic to recover
 435        from unknown notification classes.  We have four kinds of
 436        notifications: "known" ones are of a class known to the client
 437        and which match its filters; "unknown" ones are of a class that
 438        match the client's filters but that the client can't load;
 439        "tricky" ones are unserializable; and "boring" notifications
 440        are of a class that the client knows but that doesn't match its
 441        filters.  We emit NNOTIFS notifications of each kind.  We do a
 442        random shuffle on these 4*NNOTIFS notifications so it is likely
 443        that we will cover the various different cases in the logic.
 444 
 445        Specifically, what we are testing here is the logic that
 446        recovers from a fetchNotifications request that gets a
 447        ClassNotFoundException.  Because the request can contain
 448        several notifications, the client doesn't know which of them
 449        generated the exception.  So it redoes a request that asks for
 450        just one notification.  We need to be sure that this works when
 451        that one notification is of an unknown class and when it is of
 452        a known class, and in both cases when there are filtered
 453        notifications that are skipped.
 454 
 455        We are also testing the behaviour in the server when it tries
 456        to include an unserializable notification in the response to a
 457        fetchNotifications, and in the client when that happens.
 458 
 459        If the test succeeds, the listener should receive the NNOTIFS
 460        "known" notifications, and the connection listener should
 461        receive an indication of NNOTIFS lost notifications
 462        representing the "unknown" notifications.
 463 
 464        We depend on some implementation-specific features here:
 465 
 466        1. The buffer size is sufficient to contain the 4*NNOTIFS
 467        notifications which are all sent at once, before the client
 468        gets a chance to start receiving them.
 469 
 470        2. When one or more notifications are dropped because they are
 471        of unknown classes, the NOTIFS_LOST notification contains a
 472        userData that is a Long with a count of the number dropped.
 473 
 474        3. If a notification is not serializable on the server, the
 475        client gets told about it somehow, rather than having it just
 476        dropped on the floor.  The somehow might be through an RMI
 477        exception, or it might be by the server replacing the
 478        unserializable notif by a JMXConnectionNotification.NOTIFS_LOST.
 479     */
 480     private static boolean notifyTest(JMXConnector client,
 481                                       MBeanServerConnection mbsc)
 482             throws Exception {
 483         System.out.println("Send notifications including unknown ones");
 484         result = new Result();
 485         LostListener ll = new LostListener();
 486         client.addConnectionNotificationListener(ll, null, null);
 487         TestListener nl = new TestListener(ll);
 488         mbsc.addNotificationListener(on, nl, new TestFilter(), null);
 489         mbsc.invoke(on, "sendNotifs", NO_OBJECTS, NO_STRINGS);
 490 
 491         // wait for the listeners to receive all their notifs
 492         // or to fail
 493         long deadline = System.currentTimeMillis() + 60000;
 494         long remain;
 495         while ((remain = deadline - System.currentTimeMillis()) >= 0) {
 496             synchronized (result) {
 497                 if (result.failed
 498                     || (result.knownCount >= NNOTIFS
 499                         && result.lostCount >= NNOTIFS*2))
 500                     break;
 501                 result.wait(remain);
 502             }
 503         }
 504         Thread.sleep(2);  // allow any spurious extra notifs to arrive
 505         if (result.failed) {
 506             System.out.println("TEST FAILS: Notification strangeness");
 507             return false;
 508         } else if (result.knownCount == NNOTIFS
 509                    && result.lostCount == NNOTIFS*2) {
 510             System.out.println("Success: received known notifications and " +
 511                                "got NOTIFS_LOST for unknown and " +
 512                                "unserializable ones");
 513             return true;
 514         } else if (result.knownCount >= NNOTIFS
 515                 || result.lostCount >= NNOTIFS*2) {
 516             System.out.println("TEST FAILS: Received too many notifs: " +
 517                     "known=" + result.knownCount + "; lost=" + result.lostCount);
 518             return false;
 519         } else {
 520             System.out.println("TEST FAILS: Timed out without receiving " +
 521                                "all notifs: known=" + result.knownCount +
 522                                "; lost=" + result.lostCount);
 523             return false;
 524         }
 525     }
 526 
 527     public static interface TestMBean {
 528         public Object getClientUnknown() throws Exception;
 529         public void throwClientUnknown() throws Exception;
 530         public void setServerUnknown(Object o) throws Exception;
 531         public void useServerUnknown(Object o) throws Exception;
 532         public Object getUnserializable() throws Exception;
 533         public void setUnserializable(Object un) throws Exception;
 534         public void throwUnserializable() throws Exception;
 535         public void sendNotifs() throws Exception;
 536     }
 537 
 538     public static class Test extends NotificationBroadcasterSupport
 539             implements TestMBean {
 540 
 541         public Object getClientUnknown() {
 542             return clientUnknown;
 543         }
 544 
 545         public void throwClientUnknown() throws Exception {
 546             throw clientUnknown;
 547         }
 548 
 549         public void setServerUnknown(Object o) {
 550             throw new IllegalArgumentException("setServerUnknown succeeded "+
 551                                                "but should not have");
 552         }
 553 
 554         public void useServerUnknown(Object o) {
 555             throw new IllegalArgumentException("useServerUnknown succeeded "+
 556                                                "but should not have");
 557         }
 558 
 559         public Object getUnserializable() {
 560             return unserializableObject;
 561         }
 562 
 563         public void setUnserializable(Object un) {
 564             throw new IllegalArgumentException("setUnserializable succeeded " +
 565                                                "but should not have");
 566         }
 567 
 568         public void throwUnserializable() throws Exception {
 569             throw new Exception() {
 570                 private Object unserializable = unserializableObject;
 571             };
 572         }
 573 
 574         public void sendNotifs() {
 575             /* We actually send the same four notification objects
 576                NNOTIFS times each.  This doesn't particularly matter,
 577                but note that the MBeanServer will replace "this" by
 578                the sender's ObjectName the first time.  Since that's
 579                always the same, no problem.  */
 580             Notification known =
 581                 new Notification("interesting", this, 1L, 1L, "known");
 582             known.setUserData("known");
 583             Notification unknown =
 584                 new Notification("interesting", this, 1L, 1L, "unknown");
 585             unknown.setUserData(clientUnknown);
 586             Notification boring =
 587                 new Notification("boring", this, 1L, 1L, "boring");
 588             Notification tricky =
 589                 new Notification("interesting", this, 1L, 1L, "tricky");
 590             tricky.setUserData(unserializableObject);
 591 
 592             // check that the tricky notif is indeed unserializable
 593             try {
 594                 new ObjectOutputStream(new ByteArrayOutputStream())
 595                     .writeObject(tricky);
 596                 throw new RuntimeException("TEST INCORRECT: tricky notif is " +
 597                                            "serializable");
 598             } catch (NotSerializableException e) {
 599                 // OK: tricky notif is not serializable
 600             } catch (IOException e) {
 601                 throw new RuntimeException("TEST INCORRECT: tricky notif " +
 602                                             "serialization check failed");
 603             }
 604 
 605             /* Now shuffle an imaginary deck of cards where K, U, T, and
 606                B (known, unknown, tricky, boring) each appear NNOTIFS times.
 607                We explicitly seed the random number generator so we
 608                can reproduce rare test failures if necessary.  We only
 609                use a StringBuffer so we can print the shuffled deck --
 610                otherwise we could just emit the notifications as the
 611                cards are placed.  */
 612             long seed = System.currentTimeMillis();
 613             System.out.println("Random number seed is " + seed);
 614             Random r = new Random(seed);
 615             int knownCount = NNOTIFS;   // remaining K cards
 616             int unknownCount = NNOTIFS; // remaining U cards
 617             int trickyCount = NNOTIFS;  // remaining T cards
 618             int boringCount = NNOTIFS;  // remaining B cards
 619             StringBuffer notifList = new StringBuffer();
 620             for (int i = NNOTIFS * 4; i > 0; i--) {
 621                 int rand = r.nextInt(i);
 622                 // use rand to pick a card from the remaining ones
 623                 if ((rand -= knownCount) < 0) {
 624                     notifList.append('k');
 625                     knownCount--;
 626                 } else if ((rand -= unknownCount) < 0) {
 627                     notifList.append('u');
 628                     unknownCount--;
 629                 } else if ((rand -= trickyCount) < 0) {
 630                     notifList.append('t');
 631                     trickyCount--;
 632                 } else {
 633                     notifList.append('b');
 634                     boringCount--;
 635                 }
 636             }
 637             if (knownCount != 0 || unknownCount != 0
 638                 || trickyCount != 0 || boringCount != 0) {
 639                 throw new RuntimeException("TEST INCORRECT: Shuffle failed: " +
 640                                    "known=" + knownCount +" unknown=" +
 641                                    unknownCount + " tricky=" + trickyCount +
 642                                    " boring=" + boringCount +
 643                                    " deal=" + notifList);
 644             }
 645             String notifs = notifList.toString();
 646             System.out.println("Shuffle: " + notifs);
 647             for (int i = 0; i < NNOTIFS * 4; i++) {
 648                 Notification n;
 649                 switch (notifs.charAt(i)) {
 650                 case 'k': n = known; break;
 651                 case 'u': n = unknown; break;
 652                 case 't': n = tricky; break;
 653                 case 'b': n = boring; break;
 654                 default:
 655                     throw new RuntimeException("TEST INCORRECT: Bad shuffle char: " +
 656                                                notifs.charAt(i));
 657                 }
 658                 sendNotification(n);
 659             }
 660         }
 661     }
 662 
 663     private static String notificationString(Notification n) {
 664         return n.getClass().getName() + "/" + n.getType() + " \"" +
 665             n.getMessage() + "\" <" + n.getUserData() + ">";
 666     }
 667 
 668     //
 669     private static class CNListener implements NotificationListener {
 670         public void handleNotification(Notification n, Object o) {
 671             if (n instanceof JMXConnectionNotification) {
 672                 JMXConnectionNotification jn = (JMXConnectionNotification)n;
 673                 if (JMXConnectionNotification.FAILED.equals(jn.getType())) {
 674                     failed = true;
 675                 }
 676             }
 677         }
 678 
 679         public boolean failed = false;
 680     }
 681 }