1 /*
   2  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 /*
  25  * @test
  26  * @bug 6380723
  27  * @summary Decode many byte sequences in many ways
  28  * @run main/timeout=1800 FindDecoderBugs
  29  * @author Martin Buchholz
  30  */
  31 
  32 import java.util.*;
  33 import java.util.regex.*;
  34 import java.nio.*;
  35 import java.nio.charset.*;
  36 
  37 public class FindDecoderBugs {
  38 
  39     static boolean isBroken(String csn) {
  40         if (csn.equals("x-COMPOUND_TEXT")) return true;
  41         return false;
  42     }
  43 
  44     static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) {
  45         List<T> list = new ArrayList<T>(c);
  46         Collections.sort(list);
  47         return list;
  48     }
  49 
  50     static class TooManyFailures extends RuntimeException {
  51         private static final long serialVersionUID = 0L;
  52     }
  53 
  54     static String string(byte[] a) {
  55         final StringBuilder sb = new StringBuilder();
  56         for (byte b : a) {
  57             if (sb.length() != 0) sb.append(' ');
  58             sb.append(String.format("%02x", b & 0xff));
  59         }
  60         return sb.toString();
  61     }
  62 
  63     static String string(char[] a) {
  64         final StringBuilder sb = new StringBuilder();
  65         for (char c : a) {
  66             if (sb.length() != 0) sb.append(' ');
  67             sb.append(String.format("\\u%04x", (int) c));
  68         }
  69         return sb.toString();
  70     }
  71 
  72     static class Reporter {
  73         // Some machinery to make sure only a small number of errors
  74         // that are "too similar" are reported.
  75         static class Counts extends HashMap<String, Long> {
  76             private static final long serialVersionUID = -1;
  77             long inc(String signature) {
  78                 Long count = get(signature);
  79                 if (count == null) count = 0L;
  80                 put(signature, count+1);
  81                 return count+1;
  82             }
  83         }
  84 
  85         final Counts failureCounts = new Counts();
  86         final static long maxFailures = 2;
  87 
  88         final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\"");
  89         final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}");
  90 
  91         boolean bug(String format, Object... args) {
  92             String signature = String.format(format, args);
  93             signature = hideBytes.matcher(signature).replaceAll("\"??\"");
  94             signature = hideChars.matcher(signature).replaceAll("\\u????");
  95             failed++;
  96             if (failureCounts.inc(signature) <= maxFailures) {
  97                 System.out.printf(format, args);
  98                 System.out.println();
  99                 return true;
 100             }
 101             return false;
 102         }
 103 
 104         void summarize() {
 105             for (String key : sort(failureCounts.keySet()))
 106                 System.out.printf("-----%n%s%nfailures=%d%n",
 107                                   key, failureCounts.get(key));
 108         }
 109     }
 110 
 111     static final Reporter reporter = new Reporter();
 112 
 113     static class Result {
 114         final int limit;
 115         final int ipos;
 116         final boolean direct;
 117         final byte[] ia;
 118         final char[] oa;
 119         final CoderResult cr;
 120 
 121         Result(ByteBuffer ib, CharBuffer ob, CoderResult cr) {
 122             ipos = ib.position();
 123             ia = toArray(ib);
 124             oa = toArray(ob);
 125             direct = ib.isDirect();
 126             limit = ob.limit();
 127             this.cr = cr;
 128         }
 129 
 130         static byte[] toArray(ByteBuffer b) {
 131             int pos = b.position();
 132             byte[] a = new byte[b.limit()];
 133             b.position(0);
 134             b.get(a);
 135             b.position(pos);
 136             return a;
 137         }
 138 
 139         static char[] toArray(CharBuffer b) {
 140             char[] a = new char[b.position()];
 141             b.position(0);
 142             b.get(a);
 143             return a;
 144         }
 145 
 146         static boolean eq(Result x, Result y) {
 147             return x == y ||
 148                 (x != null && y != null &&
 149                  (Arrays.equals(x.oa, y.oa) &&
 150                   x.ipos == y.ipos &&
 151                   x.cr == y.cr));
 152         }
 153 
 154         public String toString() {
 155             return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s",
 156                                  string(ia), ipos, ia.length,
 157                                  cr, string(oa), oa.length, limit,
 158                                  (direct ? " (direct)" : ""));
 159         }
 160     }
 161 
 162     // legend: r=regular d=direct In=Input Ou=Output
 163     static final int maxBufSize = 20;
 164     static final ByteBuffer[] ribs = new ByteBuffer[maxBufSize];
 165     static final ByteBuffer[] dibs = new ByteBuffer[maxBufSize];
 166 
 167     static final CharBuffer[] robs = new CharBuffer[maxBufSize];
 168     static final CharBuffer[] dobs = new CharBuffer[maxBufSize];
 169     static {
 170         for (int i = 0; i < maxBufSize; i++) {
 171             ribs[i] = ByteBuffer.allocate(i);
 172             dibs[i] = ByteBuffer.allocateDirect(i);
 173             robs[i] = CharBuffer.allocate(i);
 174             dobs[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer();
 175         }
 176     }
 177 
 178     static class CharsetTester {
 179         private final Charset cs;
 180         private static final long maxFailures = 5;
 181         private long failures = 0;
 182         // private static final long maxCharsetFailures = Long.MAX_VALUE;
 183         private static final long maxCharsetFailures = 10000L;
 184         private final long failed0 = failed;
 185 
 186         CharsetTester(Charset cs) {
 187             this.cs = cs;
 188         }
 189 
 190         static boolean bug(String format, Object... args) {
 191             return reporter.bug(format, args);
 192         }
 193 
 194         Result recode(ByteBuffer ib, CharBuffer ob) {
 195             try {
 196                 char canary = '\u4242';
 197                 ib.clear();     // Prepare to read
 198                 ob.clear();     // Prepare to write
 199                 for (int i = 0; i < ob.limit(); i++)
 200                     ob.put(i, canary);
 201                 CharsetDecoder coder = cs.newDecoder();
 202                 CoderResult cr = coder.decode(ib, ob, false);
 203                 equal(ib.limit(), ib.capacity());
 204                 equal(ob.limit(), ob.capacity());
 205                 Result r = new Result(ib, ob, cr);
 206                 if (cr.isError())
 207                     check(cr.length() > 0);
 208                 if (cr.isOverflow() && ob.remaining() > 10)
 209                     bug("OVERFLOW, but there's lots of room: %s %s",
 210                         cs, r);
 211 //              if (cr.isOverflow() && ib.remaining() == 0)
 212 //                  bug("OVERFLOW, yet remaining() == 0: %s %s",
 213 //                      cs, r);
 214                 if (cr.isError() && ib.remaining() < cr.length())
 215                     bug("remaining() < CoderResult.length(): %s %s",
 216                         cs, r);
 217 //              if (ib.position() == 0 && ob.position() > 0)
 218 //                  reporter. bug("output only if input consumed: %s %s",
 219 //                                cs, r);
 220                 // Should we warn if cr.isUnmappable() ??
 221                 CoderResult cr2 = coder.decode(ib, ob, false);
 222                 if (ib.position() != r.ipos ||
 223                     ob.position() != r.oa.length ||
 224                     cr != cr2)
 225                     bug("Coding operation not idempotent: %s%n    %s%n    %s",
 226                         cs, r, new Result(ib, ob, cr2));
 227                 if (ob.position() < ob.limit() &&
 228                     ob.get(ob.position()) != canary)
 229                     bug("Buffer overrun: %s %s %s",
 230                         cs, r, ob.get(ob.position()));
 231                 return r;
 232             } catch (Throwable t) {
 233                 if (bug("Unexpected exception: %s %s %s",
 234                         cs, t.getClass().getSimpleName(),
 235                         new Result(ib, ob, null)))
 236                     t.printStackTrace();
 237                 return null;
 238             }
 239         }
 240 
 241         Result recode2(byte[] ia, int n) {
 242             int len = ia.length;
 243             ByteBuffer rib = ByteBuffer.wrap(ia);
 244             ByteBuffer dib = dibs[len];
 245             dib.clear(); dib.put(ia); dib.clear();
 246             CharBuffer rob = robs[n];
 247             CharBuffer dob = dobs[n];
 248             equal(rob.limit(), n);
 249             equal(dob.limit(), n);
 250             check(dib.isDirect());
 251             check(dob.isDirect());
 252             Result r1 = recode(rib, rob);
 253             Result r2 = recode(dib, dob);
 254             if (r1 != null && r2 != null && ! Result.eq(r1, r2))
 255                 bug("Results differ for direct buffers: %s%n    %s%n    %s",
 256                     cs, r1, r2);
 257             return r1;
 258         }
 259 
 260         Result test(byte[] ia) {
 261             if (failed - failed0 >= maxCharsetFailures)
 262                 throw new TooManyFailures();
 263 
 264             Result roomy = recode2(ia, maxBufSize - 1);
 265             if (roomy == null) return roomy;
 266             int olen = roomy.oa.length;
 267             if (olen > 0) {
 268                 if (roomy.ipos == roomy.ia.length) {
 269                     Result perfectFit = recode2(ia, olen);
 270                     if (! Result.eq(roomy, perfectFit))
 271                         bug("Results differ: %s%n    %s%n    %s",
 272                             cs, roomy, perfectFit);
 273                 }
 274                 for (int i = 0; i < olen; i++) {
 275                     Result claustrophobic = recode2(ia, i);
 276                     if (claustrophobic == null) return roomy;
 277                     if (roomy.cr.isUnderflow() &&
 278                         ! claustrophobic.cr.isOverflow())
 279                         bug("Expected OVERFLOW: %s%n    %s%n    %s",
 280                             cs, roomy, claustrophobic);
 281                 }
 282             }
 283             return roomy;
 284         }
 285 
 286         void testExhaustively(byte[] prefix, int n) {
 287             int len = prefix.length;
 288             byte[] ia = Arrays.copyOf(prefix, len + 1);
 289             for (int i = 0; i < 0x100; i++) {
 290                 ia[len] = (byte) i;
 291                 if (n == 1)
 292                     test(ia);
 293                 else
 294                     testExhaustively(ia, n - 1);
 295             }
 296         }
 297 
 298         void testRandomly(byte[] prefix, int n) {
 299             int len = prefix.length;
 300             byte[] ia = Arrays.copyOf(prefix, len + n);
 301             for (int i = 0; i < 5000; i++) {
 302                 for (int j = 0; j < n; j++)
 303                     ia[len + j] = randomByte();
 304                 test(ia);
 305             }
 306         }
 307 
 308         void testPrefix(byte[] prefix) {
 309             if (prefix.length > 0)
 310                 System.out.printf("Testing prefix %s%n", string(prefix));
 311 
 312             test(prefix);
 313 
 314             testExhaustively(prefix, 1);
 315             testExhaustively(prefix, 2);
 316             // Can you spare a week of CPU time?
 317             // testExhaustively(cs, tester, prefix, 3);
 318 
 319             testRandomly(prefix, 3);
 320             testRandomly(prefix, 4);
 321         }
 322     }
 323 
 324     private final static Random rnd = new Random();
 325     private static byte randomByte() {
 326         return (byte) rnd.nextInt(0x100);
 327     }
 328     private static byte[] randomBytes(int len) {
 329         byte[] a = new byte[len];
 330         for (int i = 0; i < len; i++)
 331             a[i] = randomByte();
 332         return a;
 333     }
 334 
 335     private static final byte SS2 = (byte) 0x8e;
 336     private static final byte SS3 = (byte) 0x8f;
 337     private static final byte ESC = (byte) 0x1b;
 338     private static final byte SO  = (byte) 0x0e;
 339     private static final byte SI  = (byte) 0x0f;
 340 
 341     private final static byte[][] stateChangers = {
 342         {SS2}, {SS3}, {SO}, {SI}
 343     };
 344 
 345     private final static byte[][]escapeSequences = {
 346         {ESC, '(', 'B'},
 347         {ESC, '(', 'I'},
 348         {ESC, '(', 'J'},
 349         {ESC, '$', '@'},
 350         {ESC, '$', 'A'},
 351         {ESC, '$', ')', 'A'},
 352         {ESC, '$', ')', 'C'},
 353         {ESC, '$', ')', 'G'},
 354         {ESC, '$', '*', 'H'},
 355         {ESC, '$', '+', 'I'},
 356         {ESC, '$', 'B'},
 357         {ESC, 'N'},
 358         {ESC, 'O'},
 359         {ESC, '$', '(', 'D'},
 360     };
 361 
 362     private static boolean isStateChanger(Charset cs, byte[] ia) {
 363         Result r = new CharsetTester(cs).recode2(ia, 9);
 364         return r == null ? false :
 365             (r.cr.isUnderflow() &&
 366              r.ipos == ia.length &&
 367              r.oa.length == 0);
 368     }
 369 
 370     private final static byte[][] incompletePrefixes = {
 371         {ESC},
 372         {ESC, '('},
 373         {ESC, '$'},
 374         {ESC, '$', '(',},
 375     };
 376 
 377     private static boolean isIncompletePrefix(Charset cs, byte[] ia) {
 378         Result r = new CharsetTester(cs).recode2(ia, 9);
 379         return r == null ? false :
 380             (r.cr.isUnderflow() &&
 381              r.ipos == 0 &&
 382              r.oa.length == 0);
 383     }
 384 
 385     private static void testCharset(Charset cs) throws Throwable {
 386         final String csn = cs.name();
 387 
 388         if (isBroken(csn)) {
 389             System.out.printf("Skipping possibly broken charset %s%n", csn);
 390             return;
 391         }
 392         System.out.println(csn);
 393         CharsetTester tester = new CharsetTester(cs);
 394 
 395         tester.testPrefix(new byte[0]);
 396 
 397         if (! csn.matches("(?:x-)?(?:UTF|JIS(?:_X)?0).*")) {
 398             for (byte[] prefix : stateChangers)
 399                 if (isStateChanger(cs, prefix))
 400                     tester.testPrefix(prefix);
 401 
 402             for (byte[] prefix : incompletePrefixes)
 403                 if (isIncompletePrefix(cs, prefix))
 404                     tester.testPrefix(prefix);
 405 
 406             if (isIncompletePrefix(cs, new byte[] {ESC}))
 407                 for (byte[] prefix : escapeSequences)
 408                     if (isStateChanger(cs, prefix))
 409                         tester.testPrefix(prefix);
 410         }
 411     }
 412 
 413     private static void realMain(String[] args) {
 414         for (Charset cs : sort(Charset.availableCharsets().values())) {
 415             try {
 416                 testCharset(cs);
 417             } catch (TooManyFailures e) {
 418                 System.out.printf("Too many failures for %s%n", cs);
 419             } catch (Throwable t) {
 420                 unexpected(t);
 421             }
 422         }
 423         reporter.summarize();
 424     }
 425 
 426     //--------------------- Infrastructure ---------------------------
 427     static volatile long passed = 0, failed = 0;
 428     static void pass() {passed++;}
 429     static void fail() {failed++; Thread.dumpStack();}
 430     static void fail(String format, Object... args) {
 431         System.out.println(String.format(format, args)); failed++;}
 432     static void fail(String msg) {System.out.println(msg); fail();}
 433     static void unexpected(Throwable t) {failed++; t.printStackTrace();}
 434     static void check(boolean cond) {if (cond) pass(); else fail();}
 435     static void equal(Object x, Object y) {
 436         if (x == null ? y == null : x.equals(y)) pass();
 437         else fail(x + " not equal to " + y);}
 438     static void equal(int x, int y) {
 439         if (x == y) pass();
 440         else fail(x + " not equal to " + y);}
 441     public static void main(String[] args) throws Throwable {
 442         try {realMain(args);} catch (Throwable t) {unexpected(t);}
 443         System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed);
 444         if (failed > 0) throw new AssertionError("Some tests failed");}
 445 }