1 /*
   2  * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 /*
  25  * @test
  26  * @bug 4915825 4921009 4934965 4977469 8019584
  27  * @summary Tests behavior when client or server gets object of unknown class
  28  * @author Eamonn McManus
  29  * @run clean MissingClassTest SingleClassLoader
  30  * @run build MissingClassTest SingleClassLoader
  31  * @run main MissingClassTest
  32  * @key randomness
  33  */
  34 
  35 /*
  36   Tests that clients and servers react correctly when they receive
  37   objects of unknown classes.  This can happen easily due to version
  38   skew or missing jar files on one end or the other.  The default
  39   behaviour of causing a connection to die because of the resultant
  40   IOException is not acceptable!  We try sending attributes and invoke
  41   parameters to the server of classes it doesn't know, and we try
  42   sending attributes, exceptions and notifications to the client of
  43   classes it doesn't know.
  44 
  45   We also test objects that are of known class but not serializable.
  46   The test cases are similar.
  47  */
  48 
  49 import java.io.ByteArrayOutputStream;
  50 import java.io.IOException;
  51 import java.io.NotSerializableException;
  52 import java.io.ObjectOutputStream;
  53 import java.net.MalformedURLException;
  54 import java.util.HashMap;
  55 import java.util.Map;
  56 import java.util.Random;
  57 import java.util.Set;
  58 import javax.management.Attribute;
  59 import javax.management.MBeanServer;
  60 import javax.management.MBeanServerConnection;
  61 import javax.management.MBeanServerFactory;
  62 import javax.management.Notification;
  63 import javax.management.NotificationBroadcasterSupport;
  64 import javax.management.NotificationFilter;
  65 import javax.management.NotificationListener;
  66 import javax.management.ObjectName;
  67 import javax.management.remote.JMXConnectionNotification;
  68 import javax.management.remote.JMXConnector;
  69 import javax.management.remote.JMXConnectorFactory;
  70 import javax.management.remote.JMXConnectorServer;
  71 import javax.management.remote.JMXConnectorServerFactory;
  72 import javax.management.remote.JMXServiceURL;
  73 import javax.management.remote.rmi.RMIConnectorServer;
  74 
  75 public class MissingClassTest {
  76     private static final int NNOTIFS = 50;
  77 
  78     private static ClassLoader clientLoader, serverLoader;
  79     private static Object serverUnknown;
  80     private static Exception clientUnknown;
  81     private static ObjectName on;
  82     private static final Object[] NO_OBJECTS = new Object[0];
  83     private static final String[] NO_STRINGS = new String[0];
  84 
  85     private static final Object unserializableObject = Thread.currentThread();
  86 
  87     private static boolean isInstance(Object o, String cn) {
  88         try {
  89             Class<?> c = Class.forName(cn);
  90             return c.isInstance(o);
  91         } catch (ClassNotFoundException x) {
  92             return false;
  93         }
  94     }
  95 
  96     public static void main(String[] args) throws Exception {
  97         System.out.println("Test that the client or server end of a " +
  98                            "connection does not fail if sent an object " +
  99                            "it cannot deserialize");
 100 
 101         on = new ObjectName("test:type=Test");
 102 
 103         ClassLoader testLoader = MissingClassTest.class.getClassLoader();
 104         clientLoader =
 105             new SingleClassLoader("$ServerUnknown$", HashMap.class,
 106                                   testLoader);
 107         serverLoader =
 108             new SingleClassLoader("$ClientUnknown$", Exception.class,
 109                                   testLoader);
 110         serverUnknown =
 111             clientLoader.loadClass("$ServerUnknown$").newInstance();
 112         clientUnknown = (Exception)
 113             serverLoader.loadClass("$ClientUnknown$").newInstance();
 114 
 115         final String[] protos = {"rmi", /*"iiop",*/ "jmxmp"};
 116         boolean ok = true;
 117         for (int i = 0; i < protos.length; i++) {
 118             try {
 119                 ok &= test(protos[i]);
 120             } catch (Exception e) {
 121                 System.out.println("TEST FAILED WITH EXCEPTION:");
 122                 e.printStackTrace(System.out);
 123                 ok = false;
 124             }
 125         }
 126 
 127         if (ok)
 128             System.out.println("Test passed");
 129         else {
 130             throw new RuntimeException("TEST FAILED");
 131         }
 132     }
 133 
 134     private static boolean test(String proto) throws Exception {
 135         System.out.println("Testing for proto " + proto);
 136 
 137         boolean ok = true;
 138 
 139         MBeanServer mbs = MBeanServerFactory.newMBeanServer();
 140         mbs.createMBean(Test.class.getName(), on);
 141 
 142         JMXConnectorServer cs;
 143         JMXServiceURL url = new JMXServiceURL(proto, null, 0);
 144         Map<String,Object> serverMap = new HashMap<>();
 145         serverMap.put(JMXConnectorServerFactory.DEFAULT_CLASS_LOADER,
 146                       serverLoader);
 147 
 148         // make sure no auto-close at server side
 149         serverMap.put("jmx.remote.x.server.connection.timeout", "888888888");
 150 
 151         try {
 152             cs = JMXConnectorServerFactory.newJMXConnectorServer(url,
 153                                                                  serverMap,
 154                                                                  mbs);
 155         } catch (MalformedURLException e) {
 156             System.out.println("System does not recognize URL: " + url +
 157                                "; ignoring");
 158             return true;
 159         }
 160         cs.start();
 161         JMXServiceURL addr = cs.getAddress();
 162         Map<String,Object> clientMap = new HashMap<>();
 163         clientMap.put(JMXConnectorFactory.DEFAULT_CLASS_LOADER,
 164                       clientLoader);
 165 
 166         System.out.println("Connecting for client-unknown test");
 167 
 168         JMXConnector client = JMXConnectorFactory.connect(addr, clientMap);
 169 
 170         // add a listener to verify no failed notif
 171         CNListener cnListener = new CNListener();
 172         client.addConnectionNotificationListener(cnListener, null, null);
 173 
 174         MBeanServerConnection mbsc = client.getMBeanServerConnection();
 175 
 176         System.out.println("Getting attribute with class unknown to client");
 177         try {
 178             Object result = mbsc.getAttribute(on, "ClientUnknown");
 179             System.out.println("TEST FAILS: getAttribute for class " +
 180                                "unknown to client should fail, returned: " +
 181                                result);
 182             ok = false;
 183         } catch (IOException e) {
 184             Throwable cause = e.getCause();
 185             if (isInstance(cause, "org.omg.CORBA.MARSHAL"))  // see CR 4935098
 186                 cause = cause.getCause();
 187             if (cause instanceof ClassNotFoundException) {
 188                 System.out.println("Success: got an IOException wrapping " +
 189                                    "a ClassNotFoundException");
 190             } else {
 191                 System.out.println("TEST FAILS: Caught IOException (" + e +
 192                                    ") but cause should be " +
 193                                    "ClassNotFoundException: " + cause);
 194                 ok = false;
 195             }
 196         }
 197 
 198         System.out.println("Doing queryNames to ensure connection alive");
 199         Set<ObjectName> names = mbsc.queryNames(null, null);
 200         System.out.println("queryNames returned " + names);
 201 
 202         System.out.println("Provoke exception of unknown class");
 203         try {
 204             mbsc.invoke(on, "throwClientUnknown", NO_OBJECTS, NO_STRINGS);
 205             System.out.println("TEST FAILS: did not get exception");
 206             ok = false;
 207         } catch (IOException e) {
 208             Throwable wrapped = e.getCause();
 209             if (isInstance(wrapped, "org.omg.CORBA.MARSHAL"))  // see CR 4935098
 210                 wrapped = wrapped.getCause();
 211             if (wrapped instanceof ClassNotFoundException) {
 212                 System.out.println("Success: got an IOException wrapping " +
 213                                    "a ClassNotFoundException: " +
 214                                    wrapped);
 215             } else {
 216                 System.out.println("TEST FAILS: Got IOException but cause " +
 217                                    "should be ClassNotFoundException: ");
 218                 if (wrapped == null)
 219                     System.out.println("(null)");
 220                 else
 221                     wrapped.printStackTrace(System.out);
 222                 ok = false;
 223             }
 224         } catch (Exception e) {
 225             System.out.println("TEST FAILS: Got wrong exception: " +
 226                                "should be IOException with cause " +
 227                                "ClassNotFoundException:");
 228             e.printStackTrace(System.out);
 229             ok = false;
 230         }
 231 
 232         System.out.println("Doing queryNames to ensure connection alive");
 233         names = mbsc.queryNames(null, null);
 234         System.out.println("queryNames returned " + names);
 235 
 236         ok &= notifyTest(client, mbsc);
 237 
 238         System.out.println("Doing queryNames to ensure connection alive");
 239         names = mbsc.queryNames(null, null);
 240         System.out.println("queryNames returned " + names);
 241 
 242         for (int i = 0; i < 2; i++) {
 243             boolean setAttribute = (i == 0); // else invoke
 244             String what = setAttribute ? "setAttribute" : "invoke";
 245             System.out.println("Trying " + what +
 246                                " with class unknown to server");
 247             try {
 248                 if (setAttribute) {
 249                     mbsc.setAttribute(on, new Attribute("ServerUnknown",
 250                                                         serverUnknown));
 251                 } else {
 252                     mbsc.invoke(on, "useServerUnknown",
 253                                 new Object[] {serverUnknown},
 254                                 new String[] {"java.lang.Object"});
 255                 }
 256                 System.out.println("TEST FAILS: " + what + " with " +
 257                                    "class unknown to server should fail " +
 258                                    "but did not");
 259                 ok = false;
 260             } catch (IOException e) {
 261                 Throwable cause = e.getCause();
 262                 if (isInstance(cause, "org.omg.CORBA.MARSHAL"))  // see CR 4935098
 263                     cause = cause.getCause();
 264                 if (cause instanceof ClassNotFoundException) {
 265                     System.out.println("Success: got an IOException " +
 266                                        "wrapping a ClassNotFoundException");
 267                 } else {
 268                     System.out.println("TEST FAILS: Caught IOException (" + e +
 269                                        ") but cause should be " +
 270                                        "ClassNotFoundException: " + cause);
 271                     e.printStackTrace(System.out); // XXX
 272                     ok = false;
 273                 }
 274             }
 275         }
 276 
 277         System.out.println("Doing queryNames to ensure connection alive");
 278         names = mbsc.queryNames(null, null);
 279         System.out.println("queryNames returned " + names);
 280 
 281         System.out.println("Trying to get unserializable attribute");
 282         try {
 283             mbsc.getAttribute(on, "Unserializable");
 284             System.out.println("TEST FAILS: get unserializable worked " +
 285                                "but should not");
 286             ok = false;
 287         } catch (IOException e) {
 288             System.out.println("Success: got an IOException: " + e +
 289                                " (cause: " + e.getCause() + ")");
 290         }
 291 
 292         System.out.println("Doing queryNames to ensure connection alive");
 293         names = mbsc.queryNames(null, null);
 294         System.out.println("queryNames returned " + names);
 295 
 296         System.out.println("Trying to set unserializable attribute");
 297         try {
 298             Attribute attr =
 299                 new Attribute("Unserializable", unserializableObject);
 300             mbsc.setAttribute(on, attr);
 301             System.out.println("TEST FAILS: set unserializable worked " +
 302                                "but should not");
 303             ok = false;
 304         } catch (IOException e) {
 305             System.out.println("Success: got an IOException: " + e +
 306                                " (cause: " + e.getCause() + ")");
 307         }
 308 
 309         System.out.println("Doing queryNames to ensure connection alive");
 310         names = mbsc.queryNames(null, null);
 311         System.out.println("queryNames returned " + names);
 312 
 313         System.out.println("Trying to throw unserializable exception");
 314         try {
 315             mbsc.invoke(on, "throwUnserializable", NO_OBJECTS, NO_STRINGS);
 316             System.out.println("TEST FAILS: throw unserializable worked " +
 317                                "but should not");
 318             ok = false;
 319         } catch (IOException e) {
 320             System.out.println("Success: got an IOException: " + e +
 321                                " (cause: " + e.getCause() + ")");
 322         }
 323 
 324         client.removeConnectionNotificationListener(cnListener);
 325         ok = ok && !cnListener.failed;
 326 
 327         client.close();
 328         cs.stop();
 329 
 330         if (ok)
 331             System.out.println("Test passed for protocol " + proto);
 332 
 333         System.out.println();
 334         return ok;
 335     }
 336 
 337     private static class TestListener implements NotificationListener {
 338         TestListener(LostListener ll) {
 339             this.lostListener = ll;
 340         }
 341 
 342         public void handleNotification(Notification n, Object h) {
 343             /* Connectors can handle unserializable notifications in
 344                one of two ways.  Either they can arrange for the
 345                client to get a NotSerializableException from its
 346                fetchNotifications call (RMI connector), or they can
 347                replace the unserializable notification by a
 348                JMXConnectionNotification.NOTIFS_LOST (JMXMP
 349                connector).  The former case is handled by code within
 350                the connector client which will end up sending a
 351                NOTIFS_LOST to our LostListener.  The logic here
 352                handles the latter case by converting it into the
 353                former.
 354              */
 355             if (n instanceof JMXConnectionNotification
 356                 && n.getType().equals(JMXConnectionNotification.NOTIFS_LOST)) {
 357                 lostListener.handleNotification(n, h);
 358                 return;
 359             }
 360 
 361             synchronized (result) {
 362                 if (!n.getType().equals("interesting")
 363                     || !n.getUserData().equals("known")) {
 364                     System.out.println("TestListener received strange notif: "
 365                                        + notificationString(n));
 366                     result.failed = true;
 367                     result.notifyAll();
 368                 } else {
 369                     result.knownCount++;
 370                     if (result.knownCount == NNOTIFS)
 371                         result.notifyAll();
 372                 }
 373             }
 374         }
 375 
 376         private LostListener lostListener;
 377     }
 378 
 379     private static class LostListener implements NotificationListener {
 380         public void handleNotification(Notification n, Object h) {
 381             synchronized (result) {
 382                 handle(n, h);
 383             }
 384         }
 385 
 386         private void handle(Notification n, Object h) {
 387             if (!(n instanceof JMXConnectionNotification)) {
 388                 System.out.println("LostListener received strange notif: " +
 389                                    notificationString(n));
 390                 result.failed = true;
 391                 result.notifyAll();
 392                 return;
 393             }
 394 
 395             JMXConnectionNotification jn = (JMXConnectionNotification) n;
 396             if (!jn.getType().equals(jn.NOTIFS_LOST)) {
 397                 System.out.println("Ignoring JMXConnectionNotification: " +
 398                                    notificationString(jn));
 399                 return;
 400             }
 401             final String msg = jn.getMessage();
 402             if ((!msg.startsWith("Dropped ")
 403                  || !msg.endsWith("classes were missing locally"))
 404                 && (!msg.startsWith("Not serializable: "))) {
 405                 System.out.println("Surprising NOTIFS_LOST getMessage: " +
 406                                    msg);
 407             }
 408             if (!(jn.getUserData() instanceof Long)) {
 409                 System.out.println("JMXConnectionNotification userData " +
 410                                    "not a Long: " + jn.getUserData());
 411                 result.failed = true;
 412             } else {
 413                 int lost = ((Long) jn.getUserData()).intValue();
 414                 result.lostCount += lost;
 415                 if (result.lostCount == NNOTIFS*2)
 416                     result.notifyAll();
 417             }
 418         }
 419     }
 420 
 421     private static class TestFilter implements NotificationFilter {
 422         public boolean isNotificationEnabled(Notification n) {
 423             return (n.getType().equals("interesting"));
 424         }
 425     }
 426 
 427     private static class Result {
 428         int knownCount, lostCount;
 429         boolean failed;
 430     }
 431     private static Result result;
 432 
 433     /* Send a bunch of notifications to exercise the logic to recover
 434        from unknown notification classes.  We have four kinds of
 435        notifications: "known" ones are of a class known to the client
 436        and which match its filters; "unknown" ones are of a class that
 437        match the client's filters but that the client can't load;
 438        "tricky" ones are unserializable; and "boring" notifications
 439        are of a class that the client knows but that doesn't match its
 440        filters.  We emit NNOTIFS notifications of each kind.  We do a
 441        random shuffle on these 4*NNOTIFS notifications so it is likely
 442        that we will cover the various different cases in the logic.
 443 
 444        Specifically, what we are testing here is the logic that
 445        recovers from a fetchNotifications request that gets a
 446        ClassNotFoundException.  Because the request can contain
 447        several notifications, the client doesn't know which of them
 448        generated the exception.  So it redoes a request that asks for
 449        just one notification.  We need to be sure that this works when
 450        that one notification is of an unknown class and when it is of
 451        a known class, and in both cases when there are filtered
 452        notifications that are skipped.
 453 
 454        We are also testing the behaviour in the server when it tries
 455        to include an unserializable notification in the response to a
 456        fetchNotifications, and in the client when that happens.
 457 
 458        If the test succeeds, the listener should receive the NNOTIFS
 459        "known" notifications, and the connection listener should
 460        receive an indication of NNOTIFS lost notifications
 461        representing the "unknown" notifications.
 462 
 463        We depend on some implementation-specific features here:
 464 
 465        1. The buffer size is sufficient to contain the 4*NNOTIFS
 466        notifications which are all sent at once, before the client
 467        gets a chance to start receiving them.
 468 
 469        2. When one or more notifications are dropped because they are
 470        of unknown classes, the NOTIFS_LOST notification contains a
 471        userData that is a Long with a count of the number dropped.
 472 
 473        3. If a notification is not serializable on the server, the
 474        client gets told about it somehow, rather than having it just
 475        dropped on the floor.  The somehow might be through an RMI
 476        exception, or it might be by the server replacing the
 477        unserializable notif by a JMXConnectionNotification.NOTIFS_LOST.
 478     */
 479     private static boolean notifyTest(JMXConnector client,
 480                                       MBeanServerConnection mbsc)
 481             throws Exception {
 482         System.out.println("Send notifications including unknown ones");
 483         result = new Result();
 484         LostListener ll = new LostListener();
 485         client.addConnectionNotificationListener(ll, null, null);
 486         TestListener nl = new TestListener(ll);
 487         mbsc.addNotificationListener(on, nl, new TestFilter(), null);
 488         mbsc.invoke(on, "sendNotifs", NO_OBJECTS, NO_STRINGS);
 489 
 490         // wait for the listeners to receive all their notifs
 491         // or to fail
 492         long deadline = System.currentTimeMillis() + 60000;
 493         long remain;
 494         while ((remain = deadline - System.currentTimeMillis()) >= 0) {
 495             synchronized (result) {
 496                 if (result.failed
 497                     || (result.knownCount >= NNOTIFS
 498                         && result.lostCount >= NNOTIFS*2))
 499                     break;
 500                 result.wait(remain);
 501             }
 502         }
 503         Thread.sleep(2);  // allow any spurious extra notifs to arrive
 504         if (result.failed) {
 505             System.out.println("TEST FAILS: Notification strangeness");
 506             return false;
 507         } else if (result.knownCount == NNOTIFS
 508                    && result.lostCount == NNOTIFS*2) {
 509             System.out.println("Success: received known notifications and " +
 510                                "got NOTIFS_LOST for unknown and " +
 511                                "unserializable ones");
 512             return true;
 513         } else if (result.knownCount >= NNOTIFS
 514                 || result.lostCount >= NNOTIFS*2) {
 515             System.out.println("TEST FAILS: Received too many notifs: " +
 516                     "known=" + result.knownCount + "; lost=" + result.lostCount);
 517             return false;
 518         } else {
 519             System.out.println("TEST FAILS: Timed out without receiving " +
 520                                "all notifs: known=" + result.knownCount +
 521                                "; lost=" + result.lostCount);
 522             return false;
 523         }
 524     }
 525 
 526     public static interface TestMBean {
 527         public Object getClientUnknown() throws Exception;
 528         public void throwClientUnknown() throws Exception;
 529         public void setServerUnknown(Object o) throws Exception;
 530         public void useServerUnknown(Object o) throws Exception;
 531         public Object getUnserializable() throws Exception;
 532         public void setUnserializable(Object un) throws Exception;
 533         public void throwUnserializable() throws Exception;
 534         public void sendNotifs() throws Exception;
 535     }
 536 
 537     public static class Test extends NotificationBroadcasterSupport
 538             implements TestMBean {
 539 
 540         public Object getClientUnknown() {
 541             return clientUnknown;
 542         }
 543 
 544         public void throwClientUnknown() throws Exception {
 545             throw clientUnknown;
 546         }
 547 
 548         public void setServerUnknown(Object o) {
 549             throw new IllegalArgumentException("setServerUnknown succeeded "+
 550                                                "but should not have");
 551         }
 552 
 553         public void useServerUnknown(Object o) {
 554             throw new IllegalArgumentException("useServerUnknown succeeded "+
 555                                                "but should not have");
 556         }
 557 
 558         public Object getUnserializable() {
 559             return unserializableObject;
 560         }
 561 
 562         public void setUnserializable(Object un) {
 563             throw new IllegalArgumentException("setUnserializable succeeded " +
 564                                                "but should not have");
 565         }
 566 
 567         public void throwUnserializable() throws Exception {
 568             throw new Exception() {
 569                 private Object unserializable = unserializableObject;
 570             };
 571         }
 572 
 573         public void sendNotifs() {
 574             /* We actually send the same four notification objects
 575                NNOTIFS times each.  This doesn't particularly matter,
 576                but note that the MBeanServer will replace "this" by
 577                the sender's ObjectName the first time.  Since that's
 578                always the same, no problem.  */
 579             Notification known =
 580                 new Notification("interesting", this, 1L, 1L, "known");
 581             known.setUserData("known");
 582             Notification unknown =
 583                 new Notification("interesting", this, 1L, 1L, "unknown");
 584             unknown.setUserData(clientUnknown);
 585             Notification boring =
 586                 new Notification("boring", this, 1L, 1L, "boring");
 587             Notification tricky =
 588                 new Notification("interesting", this, 1L, 1L, "tricky");
 589             tricky.setUserData(unserializableObject);
 590 
 591             // check that the tricky notif is indeed unserializable
 592             try {
 593                 new ObjectOutputStream(new ByteArrayOutputStream())
 594                     .writeObject(tricky);
 595                 throw new RuntimeException("TEST INCORRECT: tricky notif is " +
 596                                            "serializable");
 597             } catch (NotSerializableException e) {
 598                 // OK: tricky notif is not serializable
 599             } catch (IOException e) {
 600                 throw new RuntimeException("TEST INCORRECT: tricky notif " +
 601                                             "serialization check failed");
 602             }
 603 
 604             /* Now shuffle an imaginary deck of cards where K, U, T, and
 605                B (known, unknown, tricky, boring) each appear NNOTIFS times.
 606                We explicitly seed the random number generator so we
 607                can reproduce rare test failures if necessary.  We only
 608                use a StringBuffer so we can print the shuffled deck --
 609                otherwise we could just emit the notifications as the
 610                cards are placed.  */
 611             long seed = System.currentTimeMillis();
 612             System.out.println("Random number seed is " + seed);
 613             Random r = new Random(seed);
 614             int knownCount = NNOTIFS;   // remaining K cards
 615             int unknownCount = NNOTIFS; // remaining U cards
 616             int trickyCount = NNOTIFS;  // remaining T cards
 617             int boringCount = NNOTIFS;  // remaining B cards
 618             StringBuffer notifList = new StringBuffer();
 619             for (int i = NNOTIFS * 4; i > 0; i--) {
 620                 int rand = r.nextInt(i);
 621                 // use rand to pick a card from the remaining ones
 622                 if ((rand -= knownCount) < 0) {
 623                     notifList.append('k');
 624                     knownCount--;
 625                 } else if ((rand -= unknownCount) < 0) {
 626                     notifList.append('u');
 627                     unknownCount--;
 628                 } else if ((rand -= trickyCount) < 0) {
 629                     notifList.append('t');
 630                     trickyCount--;
 631                 } else {
 632                     notifList.append('b');
 633                     boringCount--;
 634                 }
 635             }
 636             if (knownCount != 0 || unknownCount != 0
 637                 || trickyCount != 0 || boringCount != 0) {
 638                 throw new RuntimeException("TEST INCORRECT: Shuffle failed: " +
 639                                    "known=" + knownCount +" unknown=" +
 640                                    unknownCount + " tricky=" + trickyCount +
 641                                    " boring=" + boringCount +
 642                                    " deal=" + notifList);
 643             }
 644             String notifs = notifList.toString();
 645             System.out.println("Shuffle: " + notifs);
 646             for (int i = 0; i < NNOTIFS * 4; i++) {
 647                 Notification n;
 648                 switch (notifs.charAt(i)) {
 649                 case 'k': n = known; break;
 650                 case 'u': n = unknown; break;
 651                 case 't': n = tricky; break;
 652                 case 'b': n = boring; break;
 653                 default:
 654                     throw new RuntimeException("TEST INCORRECT: Bad shuffle char: " +
 655                                                notifs.charAt(i));
 656                 }
 657                 sendNotification(n);
 658             }
 659         }
 660     }
 661 
 662     private static String notificationString(Notification n) {
 663         return n.getClass().getName() + "/" + n.getType() + " \"" +
 664             n.getMessage() + "\" <" + n.getUserData() + ">";
 665     }
 666 
 667     //
 668     private static class CNListener implements NotificationListener {
 669         public void handleNotification(Notification n, Object o) {
 670             if (n instanceof JMXConnectionNotification) {
 671                 JMXConnectionNotification jn = (JMXConnectionNotification)n;
 672                 if (JMXConnectionNotification.FAILED.equals(jn.getType())) {
 673                     failed = true;
 674                 }
 675             }
 676         }
 677 
 678         public boolean failed = false;
 679     }
 680 }