1 /*
   2  * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 /*
  25  * @test
  26  * @bug 4915825 4921009 4934965 4977469 8019584
  27  * @key randomness
  28  * @summary Tests behavior when client or server gets object of unknown class
  29  * @author Eamonn McManus
  30  *
  31  * @run clean MissingClassTest SingleClassLoader
  32  * @run build MissingClassTest SingleClassLoader
  33  * @run main MissingClassTest
  34  */
  35 
  36 /*
  37   Tests that clients and servers react correctly when they receive
  38   objects of unknown classes.  This can happen easily due to version
  39   skew or missing jar files on one end or the other.  The default
  40   behaviour of causing a connection to die because of the resultant
  41   IOException is not acceptable!  We try sending attributes and invoke
  42   parameters to the server of classes it doesn't know, and we try
  43   sending attributes, exceptions and notifications to the client of
  44   classes it doesn't know.
  45 
  46   We also test objects that are of known class but not serializable.
  47   The test cases are similar.
  48  */
  49 
  50 import java.io.ByteArrayOutputStream;
  51 import java.io.IOException;
  52 import java.io.NotSerializableException;
  53 import java.io.ObjectOutputStream;
  54 import java.net.MalformedURLException;
  55 import java.util.HashMap;
  56 import java.util.Map;
  57 import java.util.Random;
  58 import java.util.Set;
  59 import javax.management.Attribute;
  60 import javax.management.MBeanServer;
  61 import javax.management.MBeanServerConnection;
  62 import javax.management.MBeanServerFactory;
  63 import javax.management.Notification;
  64 import javax.management.NotificationBroadcasterSupport;
  65 import javax.management.NotificationFilter;
  66 import javax.management.NotificationListener;
  67 import javax.management.ObjectName;
  68 import javax.management.remote.JMXConnectionNotification;
  69 import javax.management.remote.JMXConnector;
  70 import javax.management.remote.JMXConnectorFactory;
  71 import javax.management.remote.JMXConnectorServer;
  72 import javax.management.remote.JMXConnectorServerFactory;
  73 import javax.management.remote.JMXServiceURL;
  74 import javax.management.remote.rmi.RMIConnectorServer;
  75 
  76 public class MissingClassTest {
  77     private static final int NNOTIFS = 50;
  78 
  79     private static ClassLoader clientLoader, serverLoader;
  80     private static Object serverUnknown;
  81     private static Exception clientUnknown;
  82     private static ObjectName on;
  83     private static final Object[] NO_OBJECTS = new Object[0];
  84     private static final String[] NO_STRINGS = new String[0];
  85 
  86     private static final Object unserializableObject = Thread.currentThread();
  87 
  88     private static boolean isInstance(Object o, String cn) {
  89         try {
  90             Class<?> c = Class.forName(cn);
  91             return c.isInstance(o);
  92         } catch (ClassNotFoundException x) {
  93             return false;
  94         }
  95     }
  96 
  97     public static void main(String[] args) throws Exception {
  98         System.out.println("Test that the client or server end of a " +
  99                            "connection does not fail if sent an object " +
 100                            "it cannot deserialize");
 101 
 102         on = new ObjectName("test:type=Test");
 103 
 104         ClassLoader testLoader = MissingClassTest.class.getClassLoader();
 105         clientLoader =
 106             new SingleClassLoader("$ServerUnknown$", HashMap.class,
 107                                   testLoader);
 108         serverLoader =
 109             new SingleClassLoader("$ClientUnknown$", Exception.class,
 110                                   testLoader);
 111         serverUnknown =
 112             clientLoader.loadClass("$ServerUnknown$").newInstance();
 113         clientUnknown = (Exception)
 114             serverLoader.loadClass("$ClientUnknown$").newInstance();
 115 
 116         final String[] protos = {"rmi", /*"iiop",*/ "jmxmp"};
 117         boolean ok = true;
 118         for (int i = 0; i < protos.length; i++) {
 119             try {
 120                 ok &= test(protos[i]);
 121             } catch (Exception e) {
 122                 System.out.println("TEST FAILED WITH EXCEPTION:");
 123                 e.printStackTrace(System.out);
 124                 ok = false;
 125             }
 126         }
 127 
 128         if (ok)
 129             System.out.println("Test passed");
 130         else {
 131             throw new RuntimeException("TEST FAILED");
 132         }
 133     }
 134 
 135     private static boolean test(String proto) throws Exception {
 136         System.out.println("Testing for proto " + proto);
 137 
 138         boolean ok = true;
 139 
 140         MBeanServer mbs = MBeanServerFactory.newMBeanServer();
 141         mbs.createMBean(Test.class.getName(), on);
 142 
 143         JMXConnectorServer cs;
 144         JMXServiceURL url = new JMXServiceURL(proto, null, 0);
 145         Map<String,Object> serverMap = new HashMap<>();
 146         serverMap.put(JMXConnectorServerFactory.DEFAULT_CLASS_LOADER,
 147                       serverLoader);
 148 
 149         // make sure no auto-close at server side
 150         serverMap.put("jmx.remote.x.server.connection.timeout", "888888888");
 151 
 152         try {
 153             cs = JMXConnectorServerFactory.newJMXConnectorServer(url,
 154                                                                  serverMap,
 155                                                                  mbs);
 156         } catch (MalformedURLException e) {
 157             System.out.println("System does not recognize URL: " + url +
 158                                "; ignoring");
 159             return true;
 160         }
 161         cs.start();
 162         JMXServiceURL addr = cs.getAddress();
 163         Map<String,Object> clientMap = new HashMap<>();
 164         clientMap.put(JMXConnectorFactory.DEFAULT_CLASS_LOADER,
 165                       clientLoader);
 166 
 167         System.out.println("Connecting for client-unknown test");
 168 
 169         JMXConnector client = JMXConnectorFactory.connect(addr, clientMap);
 170 
 171         // add a listener to verify no failed notif
 172         CNListener cnListener = new CNListener();
 173         client.addConnectionNotificationListener(cnListener, null, null);
 174 
 175         MBeanServerConnection mbsc = client.getMBeanServerConnection();
 176 
 177         System.out.println("Getting attribute with class unknown to client");
 178         try {
 179             Object result = mbsc.getAttribute(on, "ClientUnknown");
 180             System.out.println("TEST FAILS: getAttribute for class " +
 181                                "unknown to client should fail, returned: " +
 182                                result);
 183             ok = false;
 184         } catch (IOException e) {
 185             Throwable cause = e.getCause();
 186             if (cause instanceof ClassNotFoundException) {
 187                 System.out.println("Success: got an IOException wrapping " +
 188                                    "a ClassNotFoundException");
 189             } else {
 190                 System.out.println("TEST FAILS: Caught IOException (" + e +
 191                                    ") but cause should be " +
 192                                    "ClassNotFoundException: " + cause);
 193                 ok = false;
 194             }
 195         }
 196 
 197         System.out.println("Doing queryNames to ensure connection alive");
 198         Set<ObjectName> names = mbsc.queryNames(null, null);
 199         System.out.println("queryNames returned " + names);
 200 
 201         System.out.println("Provoke exception of unknown class");
 202         try {
 203             mbsc.invoke(on, "throwClientUnknown", NO_OBJECTS, NO_STRINGS);
 204             System.out.println("TEST FAILS: did not get exception");
 205             ok = false;
 206         } catch (IOException e) {
 207             Throwable wrapped = e.getCause();
 208             if (wrapped instanceof ClassNotFoundException) {
 209                 System.out.println("Success: got an IOException wrapping " +
 210                                    "a ClassNotFoundException: " +
 211                                    wrapped);
 212             } else {
 213                 System.out.println("TEST FAILS: Got IOException but cause " +
 214                                    "should be ClassNotFoundException: ");
 215                 if (wrapped == null)
 216                     System.out.println("(null)");
 217                 else
 218                     wrapped.printStackTrace(System.out);
 219                 ok = false;
 220             }
 221         } catch (Exception e) {
 222             System.out.println("TEST FAILS: Got wrong exception: " +
 223                                "should be IOException with cause " +
 224                                "ClassNotFoundException:");
 225             e.printStackTrace(System.out);
 226             ok = false;
 227         }
 228 
 229         System.out.println("Doing queryNames to ensure connection alive");
 230         names = mbsc.queryNames(null, null);
 231         System.out.println("queryNames returned " + names);
 232 
 233         ok &= notifyTest(client, mbsc);
 234 
 235         System.out.println("Doing queryNames to ensure connection alive");
 236         names = mbsc.queryNames(null, null);
 237         System.out.println("queryNames returned " + names);
 238 
 239         for (int i = 0; i < 2; i++) {
 240             boolean setAttribute = (i == 0); // else invoke
 241             String what = setAttribute ? "setAttribute" : "invoke";
 242             System.out.println("Trying " + what +
 243                                " with class unknown to server");
 244             try {
 245                 if (setAttribute) {
 246                     mbsc.setAttribute(on, new Attribute("ServerUnknown",
 247                                                         serverUnknown));
 248                 } else {
 249                     mbsc.invoke(on, "useServerUnknown",
 250                                 new Object[] {serverUnknown},
 251                                 new String[] {"java.lang.Object"});
 252                 }
 253                 System.out.println("TEST FAILS: " + what + " with " +
 254                                    "class unknown to server should fail " +
 255                                    "but did not");
 256                 ok = false;
 257             } catch (IOException e) {
 258                 Throwable cause = e.getCause();
 259                 if (cause instanceof ClassNotFoundException) {
 260                     System.out.println("Success: got an IOException " +
 261                                        "wrapping a ClassNotFoundException");
 262                 } else {
 263                     System.out.println("TEST FAILS: Caught IOException (" + e +
 264                                        ") but cause should be " +
 265                                        "ClassNotFoundException: " + cause);
 266                     e.printStackTrace(System.out); // XXX
 267                     ok = false;
 268                 }
 269             }
 270         }
 271 
 272         System.out.println("Doing queryNames to ensure connection alive");
 273         names = mbsc.queryNames(null, null);
 274         System.out.println("queryNames returned " + names);
 275 
 276         System.out.println("Trying to get unserializable attribute");
 277         try {
 278             mbsc.getAttribute(on, "Unserializable");
 279             System.out.println("TEST FAILS: get unserializable worked " +
 280                                "but should not");
 281             ok = false;
 282         } catch (IOException e) {
 283             System.out.println("Success: got an IOException: " + e +
 284                                " (cause: " + e.getCause() + ")");
 285         }
 286 
 287         System.out.println("Doing queryNames to ensure connection alive");
 288         names = mbsc.queryNames(null, null);
 289         System.out.println("queryNames returned " + names);
 290 
 291         System.out.println("Trying to set unserializable attribute");
 292         try {
 293             Attribute attr =
 294                 new Attribute("Unserializable", unserializableObject);
 295             mbsc.setAttribute(on, attr);
 296             System.out.println("TEST FAILS: set unserializable worked " +
 297                                "but should not");
 298             ok = false;
 299         } catch (IOException e) {
 300             System.out.println("Success: got an IOException: " + e +
 301                                " (cause: " + e.getCause() + ")");
 302         }
 303 
 304         System.out.println("Doing queryNames to ensure connection alive");
 305         names = mbsc.queryNames(null, null);
 306         System.out.println("queryNames returned " + names);
 307 
 308         System.out.println("Trying to throw unserializable exception");
 309         try {
 310             mbsc.invoke(on, "throwUnserializable", NO_OBJECTS, NO_STRINGS);
 311             System.out.println("TEST FAILS: throw unserializable worked " +
 312                                "but should not");
 313             ok = false;
 314         } catch (IOException e) {
 315             System.out.println("Success: got an IOException: " + e +
 316                                " (cause: " + e.getCause() + ")");
 317         }
 318 
 319         client.removeConnectionNotificationListener(cnListener);
 320         ok = ok && !cnListener.failed;
 321 
 322         client.close();
 323         cs.stop();
 324 
 325         if (ok)
 326             System.out.println("Test passed for protocol " + proto);
 327 
 328         System.out.println();
 329         return ok;
 330     }
 331 
 332     private static class TestListener implements NotificationListener {
 333         TestListener(LostListener ll) {
 334             this.lostListener = ll;
 335         }
 336 
 337         public void handleNotification(Notification n, Object h) {
 338             /* Connectors can handle unserializable notifications in
 339                one of two ways.  Either they can arrange for the
 340                client to get a NotSerializableException from its
 341                fetchNotifications call (RMI connector), or they can
 342                replace the unserializable notification by a
 343                JMXConnectionNotification.NOTIFS_LOST (JMXMP
 344                connector).  The former case is handled by code within
 345                the connector client which will end up sending a
 346                NOTIFS_LOST to our LostListener.  The logic here
 347                handles the latter case by converting it into the
 348                former.
 349              */
 350             if (n instanceof JMXConnectionNotification
 351                 && n.getType().equals(JMXConnectionNotification.NOTIFS_LOST)) {
 352                 lostListener.handleNotification(n, h);
 353                 return;
 354             }
 355 
 356             synchronized (result) {
 357                 if (!n.getType().equals("interesting")
 358                     || !n.getUserData().equals("known")) {
 359                     System.out.println("TestListener received strange notif: "
 360                                        + notificationString(n));
 361                     result.failed = true;
 362                     result.notifyAll();
 363                 } else {
 364                     result.knownCount++;
 365                     if (result.knownCount == NNOTIFS)
 366                         result.notifyAll();
 367                 }
 368             }
 369         }
 370 
 371         private LostListener lostListener;
 372     }
 373 
 374     private static class LostListener implements NotificationListener {
 375         public void handleNotification(Notification n, Object h) {
 376             synchronized (result) {
 377                 handle(n, h);
 378             }
 379         }
 380 
 381         private void handle(Notification n, Object h) {
 382             if (!(n instanceof JMXConnectionNotification)) {
 383                 System.out.println("LostListener received strange notif: " +
 384                                    notificationString(n));
 385                 result.failed = true;
 386                 result.notifyAll();
 387                 return;
 388             }
 389 
 390             JMXConnectionNotification jn = (JMXConnectionNotification) n;
 391             if (!jn.getType().equals(jn.NOTIFS_LOST)) {
 392                 System.out.println("Ignoring JMXConnectionNotification: " +
 393                                    notificationString(jn));
 394                 return;
 395             }
 396             final String msg = jn.getMessage();
 397             if ((!msg.startsWith("Dropped ")
 398                  || !msg.endsWith("classes were missing locally"))
 399                 && (!msg.startsWith("Not serializable: "))) {
 400                 System.out.println("Surprising NOTIFS_LOST getMessage: " +
 401                                    msg);
 402             }
 403             if (!(jn.getUserData() instanceof Long)) {
 404                 System.out.println("JMXConnectionNotification userData " +
 405                                    "not a Long: " + jn.getUserData());
 406                 result.failed = true;
 407             } else {
 408                 int lost = ((Long) jn.getUserData()).intValue();
 409                 result.lostCount += lost;
 410                 if (result.lostCount == NNOTIFS*2)
 411                     result.notifyAll();
 412             }
 413         }
 414     }
 415 
 416     private static class TestFilter implements NotificationFilter {
 417         public boolean isNotificationEnabled(Notification n) {
 418             return (n.getType().equals("interesting"));
 419         }
 420     }
 421 
 422     private static class Result {
 423         int knownCount, lostCount;
 424         boolean failed;
 425     }
 426     private static Result result;
 427 
 428     /* Send a bunch of notifications to exercise the logic to recover
 429        from unknown notification classes.  We have four kinds of
 430        notifications: "known" ones are of a class known to the client
 431        and which match its filters; "unknown" ones are of a class that
 432        match the client's filters but that the client can't load;
 433        "tricky" ones are unserializable; and "boring" notifications
 434        are of a class that the client knows but that doesn't match its
 435        filters.  We emit NNOTIFS notifications of each kind.  We do a
 436        random shuffle on these 4*NNOTIFS notifications so it is likely
 437        that we will cover the various different cases in the logic.
 438 
 439        Specifically, what we are testing here is the logic that
 440        recovers from a fetchNotifications request that gets a
 441        ClassNotFoundException.  Because the request can contain
 442        several notifications, the client doesn't know which of them
 443        generated the exception.  So it redoes a request that asks for
 444        just one notification.  We need to be sure that this works when
 445        that one notification is of an unknown class and when it is of
 446        a known class, and in both cases when there are filtered
 447        notifications that are skipped.
 448 
 449        We are also testing the behaviour in the server when it tries
 450        to include an unserializable notification in the response to a
 451        fetchNotifications, and in the client when that happens.
 452 
 453        If the test succeeds, the listener should receive the NNOTIFS
 454        "known" notifications, and the connection listener should
 455        receive an indication of NNOTIFS lost notifications
 456        representing the "unknown" notifications.
 457 
 458        We depend on some implementation-specific features here:
 459 
 460        1. The buffer size is sufficient to contain the 4*NNOTIFS
 461        notifications which are all sent at once, before the client
 462        gets a chance to start receiving them.
 463 
 464        2. When one or more notifications are dropped because they are
 465        of unknown classes, the NOTIFS_LOST notification contains a
 466        userData that is a Long with a count of the number dropped.
 467 
 468        3. If a notification is not serializable on the server, the
 469        client gets told about it somehow, rather than having it just
 470        dropped on the floor.  The somehow might be through an RMI
 471        exception, or it might be by the server replacing the
 472        unserializable notif by a JMXConnectionNotification.NOTIFS_LOST.
 473     */
 474     private static boolean notifyTest(JMXConnector client,
 475                                       MBeanServerConnection mbsc)
 476             throws Exception {
 477         System.out.println("Send notifications including unknown ones");
 478         result = new Result();
 479         LostListener ll = new LostListener();
 480         client.addConnectionNotificationListener(ll, null, null);
 481         TestListener nl = new TestListener(ll);
 482         mbsc.addNotificationListener(on, nl, new TestFilter(), null);
 483         mbsc.invoke(on, "sendNotifs", NO_OBJECTS, NO_STRINGS);
 484 
 485         // wait for the listeners to receive all their notifs
 486         // or to fail
 487         long deadline = System.currentTimeMillis() + 60000;
 488         long remain;
 489         while ((remain = deadline - System.currentTimeMillis()) >= 0) {
 490             synchronized (result) {
 491                 if (result.failed
 492                     || (result.knownCount >= NNOTIFS
 493                         && result.lostCount >= NNOTIFS*2))
 494                     break;
 495                 result.wait(remain);
 496             }
 497         }
 498         Thread.sleep(2);  // allow any spurious extra notifs to arrive
 499         if (result.failed) {
 500             System.out.println("TEST FAILS: Notification strangeness");
 501             return false;
 502         } else if (result.knownCount == NNOTIFS
 503                    && result.lostCount == NNOTIFS*2) {
 504             System.out.println("Success: received known notifications and " +
 505                                "got NOTIFS_LOST for unknown and " +
 506                                "unserializable ones");
 507             return true;
 508         } else if (result.knownCount >= NNOTIFS
 509                 || result.lostCount >= NNOTIFS*2) {
 510             System.out.println("TEST FAILS: Received too many notifs: " +
 511                     "known=" + result.knownCount + "; lost=" + result.lostCount);
 512             return false;
 513         } else {
 514             System.out.println("TEST FAILS: Timed out without receiving " +
 515                                "all notifs: known=" + result.knownCount +
 516                                "; lost=" + result.lostCount);
 517             return false;
 518         }
 519     }
 520 
 521     public static interface TestMBean {
 522         public Object getClientUnknown() throws Exception;
 523         public void throwClientUnknown() throws Exception;
 524         public void setServerUnknown(Object o) throws Exception;
 525         public void useServerUnknown(Object o) throws Exception;
 526         public Object getUnserializable() throws Exception;
 527         public void setUnserializable(Object un) throws Exception;
 528         public void throwUnserializable() throws Exception;
 529         public void sendNotifs() throws Exception;
 530     }
 531 
 532     public static class Test extends NotificationBroadcasterSupport
 533             implements TestMBean {
 534 
 535         public Object getClientUnknown() {
 536             return clientUnknown;
 537         }
 538 
 539         public void throwClientUnknown() throws Exception {
 540             throw clientUnknown;
 541         }
 542 
 543         public void setServerUnknown(Object o) {
 544             throw new IllegalArgumentException("setServerUnknown succeeded "+
 545                                                "but should not have");
 546         }
 547 
 548         public void useServerUnknown(Object o) {
 549             throw new IllegalArgumentException("useServerUnknown succeeded "+
 550                                                "but should not have");
 551         }
 552 
 553         public Object getUnserializable() {
 554             return unserializableObject;
 555         }
 556 
 557         public void setUnserializable(Object un) {
 558             throw new IllegalArgumentException("setUnserializable succeeded " +
 559                                                "but should not have");
 560         }
 561 
 562         public void throwUnserializable() throws Exception {
 563             throw new Exception() {
 564                 private Object unserializable = unserializableObject;
 565             };
 566         }
 567 
 568         public void sendNotifs() {
 569             /* We actually send the same four notification objects
 570                NNOTIFS times each.  This doesn't particularly matter,
 571                but note that the MBeanServer will replace "this" by
 572                the sender's ObjectName the first time.  Since that's
 573                always the same, no problem.  */
 574             Notification known =
 575                 new Notification("interesting", this, 1L, 1L, "known");
 576             known.setUserData("known");
 577             Notification unknown =
 578                 new Notification("interesting", this, 1L, 1L, "unknown");
 579             unknown.setUserData(clientUnknown);
 580             Notification boring =
 581                 new Notification("boring", this, 1L, 1L, "boring");
 582             Notification tricky =
 583                 new Notification("interesting", this, 1L, 1L, "tricky");
 584             tricky.setUserData(unserializableObject);
 585 
 586             // check that the tricky notif is indeed unserializable
 587             try {
 588                 new ObjectOutputStream(new ByteArrayOutputStream())
 589                     .writeObject(tricky);
 590                 throw new RuntimeException("TEST INCORRECT: tricky notif is " +
 591                                            "serializable");
 592             } catch (NotSerializableException e) {
 593                 // OK: tricky notif is not serializable
 594             } catch (IOException e) {
 595                 throw new RuntimeException("TEST INCORRECT: tricky notif " +
 596                                             "serialization check failed");
 597             }
 598 
 599             /* Now shuffle an imaginary deck of cards where K, U, T, and
 600                B (known, unknown, tricky, boring) each appear NNOTIFS times.
 601                We explicitly seed the random number generator so we
 602                can reproduce rare test failures if necessary.  We only
 603                use a StringBuffer so we can print the shuffled deck --
 604                otherwise we could just emit the notifications as the
 605                cards are placed.  */
 606             long seed = System.currentTimeMillis();
 607             System.out.println("Random number seed is " + seed);
 608             Random r = new Random(seed);
 609             int knownCount = NNOTIFS;   // remaining K cards
 610             int unknownCount = NNOTIFS; // remaining U cards
 611             int trickyCount = NNOTIFS;  // remaining T cards
 612             int boringCount = NNOTIFS;  // remaining B cards
 613             StringBuffer notifList = new StringBuffer();
 614             for (int i = NNOTIFS * 4; i > 0; i--) {
 615                 int rand = r.nextInt(i);
 616                 // use rand to pick a card from the remaining ones
 617                 if ((rand -= knownCount) < 0) {
 618                     notifList.append('k');
 619                     knownCount--;
 620                 } else if ((rand -= unknownCount) < 0) {
 621                     notifList.append('u');
 622                     unknownCount--;
 623                 } else if ((rand -= trickyCount) < 0) {
 624                     notifList.append('t');
 625                     trickyCount--;
 626                 } else {
 627                     notifList.append('b');
 628                     boringCount--;
 629                 }
 630             }
 631             if (knownCount != 0 || unknownCount != 0
 632                 || trickyCount != 0 || boringCount != 0) {
 633                 throw new RuntimeException("TEST INCORRECT: Shuffle failed: " +
 634                                    "known=" + knownCount +" unknown=" +
 635                                    unknownCount + " tricky=" + trickyCount +
 636                                    " boring=" + boringCount +
 637                                    " deal=" + notifList);
 638             }
 639             String notifs = notifList.toString();
 640             System.out.println("Shuffle: " + notifs);
 641             for (int i = 0; i < NNOTIFS * 4; i++) {
 642                 Notification n;
 643                 switch (notifs.charAt(i)) {
 644                 case 'k': n = known; break;
 645                 case 'u': n = unknown; break;
 646                 case 't': n = tricky; break;
 647                 case 'b': n = boring; break;
 648                 default:
 649                     throw new RuntimeException("TEST INCORRECT: Bad shuffle char: " +
 650                                                notifs.charAt(i));
 651                 }
 652                 sendNotification(n);
 653             }
 654         }
 655     }
 656 
 657     private static String notificationString(Notification n) {
 658         return n.getClass().getName() + "/" + n.getType() + " \"" +
 659             n.getMessage() + "\" <" + n.getUserData() + ">";
 660     }
 661 
 662     //
 663     private static class CNListener implements NotificationListener {
 664         public void handleNotification(Notification n, Object o) {
 665             if (n instanceof JMXConnectionNotification) {
 666                 JMXConnectionNotification jn = (JMXConnectionNotification)n;
 667                 if (JMXConnectionNotification.FAILED.equals(jn.getType())) {
 668                     failed = true;
 669                 }
 670             }
 671         }
 672 
 673         public boolean failed = false;
 674     }
 675 }