1 /* 2 * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 /* 25 * @test 26 * @bug 6380723 27 * @summary Decode many byte sequences in many ways (use -Dseed=X to set PRNG seed) 28 * @library /test/lib 29 * @run main/timeout=1800 FindDecoderBugs 30 * @author Martin Buchholz 31 * @key randomness 32 */ 33 34 import java.util.*; 35 import java.util.regex.*; 36 import java.nio.*; 37 import java.nio.charset.*; 38 import jdk.test.lib.RandomFactory; 39 40 public class FindDecoderBugs { 41 42 static boolean isBroken(String csn) { 43 if (csn.equals("x-COMPOUND_TEXT")) return true; 44 return false; 45 } 46 47 static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) { 48 List<T> list = new ArrayList<T>(c); 49 Collections.sort(list); 50 return list; 51 } 52 53 static class TooManyFailures extends RuntimeException { 54 private static final long serialVersionUID = 0L; 55 } 56 57 static String string(byte[] a) { 58 final StringBuilder sb = new StringBuilder(); 59 for (byte b : a) { 60 if (sb.length() != 0) sb.append(' '); 61 sb.append(String.format("%02x", b & 0xff)); 62 } 63 return sb.toString(); 64 } 65 66 static String string(char[] a) { 67 final StringBuilder sb = new StringBuilder(); 68 for (char c : a) { 69 if (sb.length() != 0) sb.append(' '); 70 sb.append(String.format("\\u%04x", (int) c)); 71 } 72 return sb.toString(); 73 } 74 75 static class Reporter { 76 // Some machinery to make sure only a small number of errors 77 // that are "too similar" are reported. 78 static class Counts extends HashMap<String, Long> { 79 private static final long serialVersionUID = -1; 80 long inc(String signature) { 81 Long count = get(signature); 82 if (count == null) count = 0L; 83 put(signature, count+1); 84 return count+1; 85 } 86 } 87 88 final Counts failureCounts = new Counts(); 89 final static long maxFailures = 2; 90 91 final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\""); 92 final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}"); 93 94 boolean bug(String format, Object... args) { 95 String signature = String.format(format, args); 96 signature = hideBytes.matcher(signature).replaceAll("\"??\""); 97 signature = hideChars.matcher(signature).replaceAll("\\u????"); 98 failed++; 99 if (failureCounts.inc(signature) <= maxFailures) { 100 System.out.printf(format, args); 101 System.out.println(); 102 return true; 103 } 104 return false; 105 } 106 107 void summarize() { 108 for (String key : sort(failureCounts.keySet())) 109 System.out.printf("-----%n%s%nfailures=%d%n", 110 key, failureCounts.get(key)); 111 } 112 } 113 114 static final Reporter reporter = new Reporter(); 115 116 static class Result { 117 final int limit; 118 final int ipos; 119 final boolean direct; 120 final byte[] ia; 121 final char[] oa; 122 final CoderResult cr; 123 124 Result(ByteBuffer ib, CharBuffer ob, CoderResult cr) { 125 ipos = ib.position(); 126 ia = toArray(ib); 127 oa = toArray(ob); 128 direct = ib.isDirect(); 129 limit = ob.limit(); 130 this.cr = cr; 131 } 132 133 static byte[] toArray(ByteBuffer b) { 134 int pos = b.position(); 135 byte[] a = new byte[b.limit()]; 136 b.position(0); 137 b.get(a); 138 b.position(pos); 139 return a; 140 } 141 142 static char[] toArray(CharBuffer b) { 143 char[] a = new char[b.position()]; 144 b.position(0); 145 b.get(a); 146 return a; 147 } 148 149 static boolean eq(Result x, Result y) { 150 return x == y || 151 (x != null && y != null && 152 (Arrays.equals(x.oa, y.oa) && 153 x.ipos == y.ipos && 154 x.cr == y.cr)); 155 } 156 157 public String toString() { 158 return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s", 159 string(ia), ipos, ia.length, 160 cr, string(oa), oa.length, limit, 161 (direct ? " (direct)" : "")); 162 } 163 } 164 165 // legend: r=regular d=direct In=Input Ou=Output 166 static final int maxBufSize = 20; 167 static final ByteBuffer[] ribs = new ByteBuffer[maxBufSize]; 168 static final ByteBuffer[] dibs = new ByteBuffer[maxBufSize]; 169 170 static final CharBuffer[] robs = new CharBuffer[maxBufSize]; 171 static final CharBuffer[] dobs = new CharBuffer[maxBufSize]; 172 static { 173 for (int i = 0; i < maxBufSize; i++) { 174 ribs[i] = ByteBuffer.allocate(i); 175 dibs[i] = ByteBuffer.allocateDirect(i); 176 robs[i] = CharBuffer.allocate(i); 177 dobs[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer(); 178 } 179 } 180 181 static class CharsetTester { 182 private final Charset cs; 183 private static final long maxFailures = 5; 184 private long failures = 0; 185 // private static final long maxCharsetFailures = Long.MAX_VALUE; 186 private static final long maxCharsetFailures = 10000L; 187 private final long failed0 = failed; 188 189 CharsetTester(Charset cs) { 190 this.cs = cs; 191 } 192 193 static boolean bug(String format, Object... args) { 194 return reporter.bug(format, args); 195 } 196 197 Result recode(ByteBuffer ib, CharBuffer ob) { 198 try { 199 char canary = '\u4242'; 200 ib.clear(); // Prepare to read 201 ob.clear(); // Prepare to write 202 for (int i = 0; i < ob.limit(); i++) 203 ob.put(i, canary); 204 CharsetDecoder coder = cs.newDecoder(); 205 CoderResult cr = coder.decode(ib, ob, false); 206 equal(ib.limit(), ib.capacity()); 207 equal(ob.limit(), ob.capacity()); 208 Result r = new Result(ib, ob, cr); 209 if (cr.isError()) 210 check(cr.length() > 0); 211 if (cr.isOverflow() && ob.remaining() > 10) 212 bug("OVERFLOW, but there's lots of room: %s %s", 213 cs, r); 214 // if (cr.isOverflow() && ib.remaining() == 0) 215 // bug("OVERFLOW, yet remaining() == 0: %s %s", 216 // cs, r); 217 if (cr.isError() && ib.remaining() < cr.length()) 218 bug("remaining() < CoderResult.length(): %s %s", 219 cs, r); 220 // if (ib.position() == 0 && ob.position() > 0) 221 // reporter. bug("output only if input consumed: %s %s", 222 // cs, r); 223 // Should we warn if cr.isUnmappable() ?? 224 CoderResult cr2 = coder.decode(ib, ob, false); 225 if (ib.position() != r.ipos || 226 ob.position() != r.oa.length || 227 cr != cr2) 228 bug("Coding operation not idempotent: %s%n %s%n %s", 229 cs, r, new Result(ib, ob, cr2)); 230 if (ob.position() < ob.limit() && 231 ob.get(ob.position()) != canary) 232 bug("Buffer overrun: %s %s %s", 233 cs, r, ob.get(ob.position())); 234 return r; 235 } catch (Throwable t) { 236 if (bug("Unexpected exception: %s %s %s", 237 cs, t.getClass().getSimpleName(), 238 new Result(ib, ob, null))) 239 t.printStackTrace(); 240 return null; 241 } 242 } 243 244 Result recode2(byte[] ia, int n) { 245 int len = ia.length; 246 ByteBuffer rib = ByteBuffer.wrap(ia); 247 ByteBuffer dib = dibs[len]; 248 dib.clear(); dib.put(ia); dib.clear(); 249 CharBuffer rob = robs[n]; 250 CharBuffer dob = dobs[n]; 251 equal(rob.limit(), n); 252 equal(dob.limit(), n); 253 check(dib.isDirect()); 254 check(dob.isDirect()); 255 Result r1 = recode(rib, rob); 256 Result r2 = recode(dib, dob); 257 if (r1 != null && r2 != null && ! Result.eq(r1, r2)) 258 bug("Results differ for direct buffers: %s%n %s%n %s", 259 cs, r1, r2); 260 return r1; 261 } 262 263 Result test(byte[] ia) { 264 if (failed - failed0 >= maxCharsetFailures) 265 throw new TooManyFailures(); 266 267 Result roomy = recode2(ia, maxBufSize - 1); 268 if (roomy == null) return roomy; 269 int olen = roomy.oa.length; 270 if (olen > 0) { 271 if (roomy.ipos == roomy.ia.length) { 272 Result perfectFit = recode2(ia, olen); 273 if (! Result.eq(roomy, perfectFit)) 274 bug("Results differ: %s%n %s%n %s", 275 cs, roomy, perfectFit); 276 } 277 for (int i = 0; i < olen; i++) { 278 Result claustrophobic = recode2(ia, i); 279 if (claustrophobic == null) return roomy; 280 if (roomy.cr.isUnderflow() && 281 ! claustrophobic.cr.isOverflow()) 282 bug("Expected OVERFLOW: %s%n %s%n %s", 283 cs, roomy, claustrophobic); 284 } 285 } 286 return roomy; 287 } 288 289 void testExhaustively(byte[] prefix, int n) { 290 int len = prefix.length; 291 byte[] ia = Arrays.copyOf(prefix, len + 1); 292 for (int i = 0; i < 0x100; i++) { 293 ia[len] = (byte) i; 294 if (n == 1) 295 test(ia); 296 else 297 testExhaustively(ia, n - 1); 298 } 299 } 300 301 void testRandomly(byte[] prefix, int n) { 302 int len = prefix.length; 303 byte[] ia = Arrays.copyOf(prefix, len + n); 304 for (int i = 0; i < 5000; i++) { 305 for (int j = 0; j < n; j++) 306 ia[len + j] = randomByte(); 307 test(ia); 308 } 309 } 310 311 void testPrefix(byte[] prefix) { 312 if (prefix.length > 0) 313 System.out.printf("Testing prefix %s%n", string(prefix)); 314 315 test(prefix); 316 317 testExhaustively(prefix, 1); 318 testExhaustively(prefix, 2); 319 // Can you spare a week of CPU time? 320 // testExhaustively(cs, tester, prefix, 3); 321 322 testRandomly(prefix, 3); 323 testRandomly(prefix, 4); 324 } 325 } 326 327 private final static Random rnd = RandomFactory.getRandom(); 328 private static byte randomByte() { 329 return (byte) rnd.nextInt(0x100); 330 } 331 private static byte[] randomBytes(int len) { 332 byte[] a = new byte[len]; 333 for (int i = 0; i < len; i++) 334 a[i] = randomByte(); 335 return a; 336 } 337 338 private static final byte SS2 = (byte) 0x8e; 339 private static final byte SS3 = (byte) 0x8f; 340 private static final byte ESC = (byte) 0x1b; 341 private static final byte SO = (byte) 0x0e; 342 private static final byte SI = (byte) 0x0f; 343 344 private final static byte[][] stateChangers = { 345 {SS2}, {SS3}, {SO}, {SI} 346 }; 347 348 private final static byte[][]escapeSequences = { 349 {ESC, '(', 'B'}, 350 {ESC, '(', 'I'}, 351 {ESC, '(', 'J'}, 352 {ESC, '$', '@'}, 353 {ESC, '$', 'A'}, 354 {ESC, '$', ')', 'A'}, 355 {ESC, '$', ')', 'C'}, 356 {ESC, '$', ')', 'G'}, 357 {ESC, '$', '*', 'H'}, 358 {ESC, '$', '+', 'I'}, 359 {ESC, '$', 'B'}, 360 {ESC, 'N'}, 361 {ESC, 'O'}, 362 {ESC, '$', '(', 'D'}, 363 }; 364 365 private static boolean isStateChanger(Charset cs, byte[] ia) { 366 Result r = new CharsetTester(cs).recode2(ia, 9); 367 return r == null ? false : 368 (r.cr.isUnderflow() && 369 r.ipos == ia.length && 370 r.oa.length == 0); 371 } 372 373 private final static byte[][] incompletePrefixes = { 374 {ESC}, 375 {ESC, '('}, 376 {ESC, '$'}, 377 {ESC, '$', '(',}, 378 }; 379 380 private static boolean isIncompletePrefix(Charset cs, byte[] ia) { 381 Result r = new CharsetTester(cs).recode2(ia, 9); 382 return r == null ? false : 383 (r.cr.isUnderflow() && 384 r.ipos == 0 && 385 r.oa.length == 0); 386 } 387 388 private static void testCharset(Charset cs) throws Throwable { 389 final String csn = cs.name(); 390 391 if (isBroken(csn)) { 392 System.out.printf("Skipping possibly broken charset %s%n", csn); 393 return; 394 } 395 System.out.println(csn); 396 CharsetTester tester = new CharsetTester(cs); 397 398 tester.testPrefix(new byte[0]); 399 400 if (! csn.matches("(?:x-)?(?:UTF|JIS(?:_X)?0).*")) { 401 for (byte[] prefix : stateChangers) 402 if (isStateChanger(cs, prefix)) 403 tester.testPrefix(prefix); 404 405 for (byte[] prefix : incompletePrefixes) 406 if (isIncompletePrefix(cs, prefix)) 407 tester.testPrefix(prefix); 408 409 if (isIncompletePrefix(cs, new byte[] {ESC})) 410 for (byte[] prefix : escapeSequences) 411 if (isStateChanger(cs, prefix)) 412 tester.testPrefix(prefix); 413 } 414 } 415 416 private static void realMain(String[] args) { 417 for (Charset cs : sort(Charset.availableCharsets().values())) { 418 try { 419 testCharset(cs); 420 } catch (TooManyFailures e) { 421 System.out.printf("Too many failures for %s%n", cs); 422 } catch (Throwable t) { 423 unexpected(t); 424 } 425 } 426 reporter.summarize(); 427 } 428 429 //--------------------- Infrastructure --------------------------- 430 static volatile long passed = 0, failed = 0; 431 static void pass() {passed++;} 432 static void fail() {failed++; Thread.dumpStack();} 433 static void fail(String format, Object... args) { 434 System.out.println(String.format(format, args)); failed++;} 435 static void fail(String msg) {System.out.println(msg); fail();} 436 static void unexpected(Throwable t) {failed++; t.printStackTrace();} 437 static void check(boolean cond) {if (cond) pass(); else fail();} 438 static void equal(Object x, Object y) { 439 if (x == null ? y == null : x.equals(y)) pass(); 440 else fail(x + " not equal to " + y);} 441 static void equal(int x, int y) { 442 if (x == y) pass(); 443 else fail(x + " not equal to " + y);} 444 public static void main(String[] args) throws Throwable { 445 try {realMain(args);} catch (Throwable t) {unexpected(t);} 446 System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed); 447 if (failed > 0) throw new AssertionError("Some tests failed");} 448 }