1 /* 2 * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 /* 25 * @test 26 * @bug 6380723 27 * @summary Decode many byte sequences in many ways (use -Dseed=X to set PRNG seed) 28 * @library /lib/testlibrary 29 * @build jdk.testlibrary.* 30 * @run main/timeout=1800 FindDecoderBugs 31 * @author Martin Buchholz 32 * @key intermittent randomness 33 */ 34 35 import java.util.*; 36 import java.util.regex.*; 37 import java.nio.*; 38 import java.nio.charset.*; 39 import jdk.testlibrary.RandomFactory; 40 41 public class FindDecoderBugs { 42 43 static boolean isBroken(String csn) { 44 if (csn.equals("x-COMPOUND_TEXT")) return true; 45 return false; 46 } 47 48 static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) { 49 List<T> list = new ArrayList<T>(c); 50 Collections.sort(list); 51 return list; 52 } 53 54 static class TooManyFailures extends RuntimeException { 55 private static final long serialVersionUID = 0L; 56 } 57 58 static String string(byte[] a) { 59 final StringBuilder sb = new StringBuilder(); 60 for (byte b : a) { 61 if (sb.length() != 0) sb.append(' '); 62 sb.append(String.format("%02x", b & 0xff)); 63 } 64 return sb.toString(); 65 } 66 67 static String string(char[] a) { 68 final StringBuilder sb = new StringBuilder(); 69 for (char c : a) { 70 if (sb.length() != 0) sb.append(' '); 71 sb.append(String.format("\\u%04x", (int) c)); 72 } 73 return sb.toString(); 74 } 75 76 static class Reporter { 77 // Some machinery to make sure only a small number of errors 78 // that are "too similar" are reported. 79 static class Counts extends HashMap<String, Long> { 80 private static final long serialVersionUID = -1; 81 long inc(String signature) { 82 Long count = get(signature); 83 if (count == null) count = 0L; 84 put(signature, count+1); 85 return count+1; 86 } 87 } 88 89 final Counts failureCounts = new Counts(); 90 final static long maxFailures = 2; 91 92 final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\""); 93 final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}"); 94 95 boolean bug(String format, Object... args) { 96 String signature = String.format(format, args); 97 signature = hideBytes.matcher(signature).replaceAll("\"??\""); 98 signature = hideChars.matcher(signature).replaceAll("\\u????"); 99 failed++; 100 if (failureCounts.inc(signature) <= maxFailures) { 101 System.out.printf(format, args); 102 System.out.println(); 103 return true; 104 } 105 return false; 106 } 107 108 void summarize() { 109 for (String key : sort(failureCounts.keySet())) 110 System.out.printf("-----%n%s%nfailures=%d%n", 111 key, failureCounts.get(key)); 112 } 113 } 114 115 static final Reporter reporter = new Reporter(); 116 117 static class Result { 118 final int limit; 119 final int ipos; 120 final boolean direct; 121 final byte[] ia; 122 final char[] oa; 123 final CoderResult cr; 124 125 Result(ByteBuffer ib, CharBuffer ob, CoderResult cr) { 126 ipos = ib.position(); 127 ia = toArray(ib); 128 oa = toArray(ob); 129 direct = ib.isDirect(); 130 limit = ob.limit(); 131 this.cr = cr; 132 } 133 134 static byte[] toArray(ByteBuffer b) { 135 int pos = b.position(); 136 byte[] a = new byte[b.limit()]; 137 b.position(0); 138 b.get(a); 139 b.position(pos); 140 return a; 141 } 142 143 static char[] toArray(CharBuffer b) { 144 char[] a = new char[b.position()]; 145 b.position(0); 146 b.get(a); 147 return a; 148 } 149 150 static boolean eq(Result x, Result y) { 151 return x == y || 152 (x != null && y != null && 153 (Arrays.equals(x.oa, y.oa) && 154 x.ipos == y.ipos && 155 x.cr == y.cr)); 156 } 157 158 public String toString() { 159 return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s", 160 string(ia), ipos, ia.length, 161 cr, string(oa), oa.length, limit, 162 (direct ? " (direct)" : "")); 163 } 164 } 165 166 // legend: r=regular d=direct In=Input Ou=Output 167 static final int maxBufSize = 20; 168 static final ByteBuffer[] ribs = new ByteBuffer[maxBufSize]; 169 static final ByteBuffer[] dibs = new ByteBuffer[maxBufSize]; 170 171 static final CharBuffer[] robs = new CharBuffer[maxBufSize]; 172 static final CharBuffer[] dobs = new CharBuffer[maxBufSize]; 173 static { 174 for (int i = 0; i < maxBufSize; i++) { 175 ribs[i] = ByteBuffer.allocate(i); 176 dibs[i] = ByteBuffer.allocateDirect(i); 177 robs[i] = CharBuffer.allocate(i); 178 dobs[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer(); 179 } 180 } 181 182 static class CharsetTester { 183 private final Charset cs; 184 private static final long maxFailures = 5; 185 private long failures = 0; 186 // private static final long maxCharsetFailures = Long.MAX_VALUE; 187 private static final long maxCharsetFailures = 10000L; 188 private final long failed0 = failed; 189 190 CharsetTester(Charset cs) { 191 this.cs = cs; 192 } 193 194 static boolean bug(String format, Object... args) { 195 return reporter.bug(format, args); 196 } 197 198 Result recode(ByteBuffer ib, CharBuffer ob) { 199 try { 200 char canary = '\u4242'; 201 ib.clear(); // Prepare to read 202 ob.clear(); // Prepare to write 203 for (int i = 0; i < ob.limit(); i++) 204 ob.put(i, canary); 205 CharsetDecoder coder = cs.newDecoder(); 206 CoderResult cr = coder.decode(ib, ob, false); 207 equal(ib.limit(), ib.capacity()); 208 equal(ob.limit(), ob.capacity()); 209 Result r = new Result(ib, ob, cr); 210 if (cr.isError()) 211 check(cr.length() > 0); 212 if (cr.isOverflow() && ob.remaining() > 10) 213 bug("OVERFLOW, but there's lots of room: %s %s", 214 cs, r); 215 // if (cr.isOverflow() && ib.remaining() == 0) 216 // bug("OVERFLOW, yet remaining() == 0: %s %s", 217 // cs, r); 218 if (cr.isError() && ib.remaining() < cr.length()) 219 bug("remaining() < CoderResult.length(): %s %s", 220 cs, r); 221 // if (ib.position() == 0 && ob.position() > 0) 222 // reporter. bug("output only if input consumed: %s %s", 223 // cs, r); 224 // Should we warn if cr.isUnmappable() ?? 225 CoderResult cr2 = coder.decode(ib, ob, false); 226 if (ib.position() != r.ipos || 227 ob.position() != r.oa.length || 228 cr != cr2) 229 bug("Coding operation not idempotent: %s%n %s%n %s", 230 cs, r, new Result(ib, ob, cr2)); 231 if (ob.position() < ob.limit() && 232 ob.get(ob.position()) != canary) 233 bug("Buffer overrun: %s %s %s", 234 cs, r, ob.get(ob.position())); 235 return r; 236 } catch (Throwable t) { 237 if (bug("Unexpected exception: %s %s %s", 238 cs, t.getClass().getSimpleName(), 239 new Result(ib, ob, null))) 240 t.printStackTrace(); 241 return null; 242 } 243 } 244 245 Result recode2(byte[] ia, int n) { 246 int len = ia.length; 247 ByteBuffer rib = ByteBuffer.wrap(ia); 248 ByteBuffer dib = dibs[len]; 249 dib.clear(); dib.put(ia); dib.clear(); 250 CharBuffer rob = robs[n]; 251 CharBuffer dob = dobs[n]; 252 equal(rob.limit(), n); 253 equal(dob.limit(), n); 254 check(dib.isDirect()); 255 check(dob.isDirect()); 256 Result r1 = recode(rib, rob); 257 Result r2 = recode(dib, dob); 258 if (r1 != null && r2 != null && ! Result.eq(r1, r2)) 259 bug("Results differ for direct buffers: %s%n %s%n %s", 260 cs, r1, r2); 261 return r1; 262 } 263 264 Result test(byte[] ia) { 265 if (failed - failed0 >= maxCharsetFailures) 266 throw new TooManyFailures(); 267 268 Result roomy = recode2(ia, maxBufSize - 1); 269 if (roomy == null) return roomy; 270 int olen = roomy.oa.length; 271 if (olen > 0) { 272 if (roomy.ipos == roomy.ia.length) { 273 Result perfectFit = recode2(ia, olen); 274 if (! Result.eq(roomy, perfectFit)) 275 bug("Results differ: %s%n %s%n %s", 276 cs, roomy, perfectFit); 277 } 278 for (int i = 0; i < olen; i++) { 279 Result claustrophobic = recode2(ia, i); 280 if (claustrophobic == null) return roomy; 281 if (roomy.cr.isUnderflow() && 282 ! claustrophobic.cr.isOverflow()) 283 bug("Expected OVERFLOW: %s%n %s%n %s", 284 cs, roomy, claustrophobic); 285 } 286 } 287 return roomy; 288 } 289 290 void testExhaustively(byte[] prefix, int n) { 291 int len = prefix.length; 292 byte[] ia = Arrays.copyOf(prefix, len + 1); 293 for (int i = 0; i < 0x100; i++) { 294 ia[len] = (byte) i; 295 if (n == 1) 296 test(ia); 297 else 298 testExhaustively(ia, n - 1); 299 } 300 } 301 302 void testRandomly(byte[] prefix, int n) { 303 int len = prefix.length; 304 byte[] ia = Arrays.copyOf(prefix, len + n); 305 for (int i = 0; i < 5000; i++) { 306 for (int j = 0; j < n; j++) 307 ia[len + j] = randomByte(); 308 test(ia); 309 } 310 } 311 312 void testPrefix(byte[] prefix) { 313 if (prefix.length > 0) 314 System.out.printf("Testing prefix %s%n", string(prefix)); 315 316 test(prefix); 317 318 testExhaustively(prefix, 1); 319 testExhaustively(prefix, 2); 320 // Can you spare a week of CPU time? 321 // testExhaustively(cs, tester, prefix, 3); 322 323 testRandomly(prefix, 3); 324 testRandomly(prefix, 4); 325 } 326 } 327 328 private final static Random rnd = RandomFactory.getRandom(); 329 private static byte randomByte() { 330 return (byte) rnd.nextInt(0x100); 331 } 332 private static byte[] randomBytes(int len) { 333 byte[] a = new byte[len]; 334 for (int i = 0; i < len; i++) 335 a[i] = randomByte(); 336 return a; 337 } 338 339 private static final byte SS2 = (byte) 0x8e; 340 private static final byte SS3 = (byte) 0x8f; 341 private static final byte ESC = (byte) 0x1b; 342 private static final byte SO = (byte) 0x0e; 343 private static final byte SI = (byte) 0x0f; 344 345 private final static byte[][] stateChangers = { 346 {SS2}, {SS3}, {SO}, {SI} 347 }; 348 349 private final static byte[][]escapeSequences = { 350 {ESC, '(', 'B'}, 351 {ESC, '(', 'I'}, 352 {ESC, '(', 'J'}, 353 {ESC, '$', '@'}, 354 {ESC, '$', 'A'}, 355 {ESC, '$', ')', 'A'}, 356 {ESC, '$', ')', 'C'}, 357 {ESC, '$', ')', 'G'}, 358 {ESC, '$', '*', 'H'}, 359 {ESC, '$', '+', 'I'}, 360 {ESC, '$', 'B'}, 361 {ESC, 'N'}, 362 {ESC, 'O'}, 363 {ESC, '$', '(', 'D'}, 364 }; 365 366 private static boolean isStateChanger(Charset cs, byte[] ia) { 367 Result r = new CharsetTester(cs).recode2(ia, 9); 368 return r == null ? false : 369 (r.cr.isUnderflow() && 370 r.ipos == ia.length && 371 r.oa.length == 0); 372 } 373 374 private final static byte[][] incompletePrefixes = { 375 {ESC}, 376 {ESC, '('}, 377 {ESC, '$'}, 378 {ESC, '$', '(',}, 379 }; 380 381 private static boolean isIncompletePrefix(Charset cs, byte[] ia) { 382 Result r = new CharsetTester(cs).recode2(ia, 9); 383 return r == null ? false : 384 (r.cr.isUnderflow() && 385 r.ipos == 0 && 386 r.oa.length == 0); 387 } 388 389 private static void testCharset(Charset cs) throws Throwable { 390 final String csn = cs.name(); 391 392 if (isBroken(csn)) { 393 System.out.printf("Skipping possibly broken charset %s%n", csn); 394 return; 395 } 396 System.out.println(csn); 397 CharsetTester tester = new CharsetTester(cs); 398 399 tester.testPrefix(new byte[0]); 400 401 if (! csn.matches("(?:x-)?(?:UTF|JIS(?:_X)?0).*")) { 402 for (byte[] prefix : stateChangers) 403 if (isStateChanger(cs, prefix)) 404 tester.testPrefix(prefix); 405 406 for (byte[] prefix : incompletePrefixes) 407 if (isIncompletePrefix(cs, prefix)) 408 tester.testPrefix(prefix); 409 410 if (isIncompletePrefix(cs, new byte[] {ESC})) 411 for (byte[] prefix : escapeSequences) 412 if (isStateChanger(cs, prefix)) 413 tester.testPrefix(prefix); 414 } 415 } 416 417 private static void realMain(String[] args) { 418 for (Charset cs : sort(Charset.availableCharsets().values())) { 419 try { 420 testCharset(cs); 421 } catch (TooManyFailures e) { 422 System.out.printf("Too many failures for %s%n", cs); 423 } catch (Throwable t) { 424 unexpected(t); 425 } 426 } 427 reporter.summarize(); 428 } 429 430 //--------------------- Infrastructure --------------------------- 431 static volatile long passed = 0, failed = 0; 432 static void pass() {passed++;} 433 static void fail() {failed++; Thread.dumpStack();} 434 static void fail(String format, Object... args) { 435 System.out.println(String.format(format, args)); failed++;} 436 static void fail(String msg) {System.out.println(msg); fail();} 437 static void unexpected(Throwable t) {failed++; t.printStackTrace();} 438 static void check(boolean cond) {if (cond) pass(); else fail();} 439 static void equal(Object x, Object y) { 440 if (x == null ? y == null : x.equals(y)) pass(); 441 else fail(x + " not equal to " + y);} 442 static void equal(int x, int y) { 443 if (x == y) pass(); 444 else fail(x + " not equal to " + y);} 445 public static void main(String[] args) throws Throwable { 446 try {realMain(args);} catch (Throwable t) {unexpected(t);} 447 System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed); 448 if (failed > 0) throw new AssertionError("Some tests failed");} 449 }