1 /* 2 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 /* 25 * @test 26 * @bug 6380723 27 * @summary Decode many byte sequences in many ways 28 * @run main/timeout=1800 FindDecoderBugs 29 * @author Martin Buchholz 30 */ 31 32 import java.util.*; 33 import java.util.regex.*; 34 import java.nio.*; 35 import java.nio.charset.*; 36 37 public class FindDecoderBugs { 38 39 static boolean isBroken(String csn) { 40 if (csn.equals("x-COMPOUND_TEXT")) return true; 41 return false; 42 } 43 44 static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) { 45 List<T> list = new ArrayList<T>(c); 46 Collections.sort(list); 47 return list; 48 } 49 50 static class TooManyFailures extends RuntimeException { 51 private static final long serialVersionUID = 0L; 52 } 53 54 static String string(byte[] a) { 55 final StringBuilder sb = new StringBuilder(); 56 for (byte b : a) { 57 if (sb.length() != 0) sb.append(' '); 58 sb.append(String.format("%02x", b & 0xff)); 59 } 60 return sb.toString(); 61 } 62 63 static String string(char[] a) { 64 final StringBuilder sb = new StringBuilder(); 65 for (char c : a) { 66 if (sb.length() != 0) sb.append(' '); 67 sb.append(String.format("\\u%04x", (int) c)); 68 } 69 return sb.toString(); 70 } 71 72 static class Reporter { 73 // Some machinery to make sure only a small number of errors 74 // that are "too similar" are reported. 75 static class Counts extends HashMap<String, Long> { 76 private static final long serialVersionUID = -1; 77 long inc(String signature) { 78 Long count = get(signature); 79 if (count == null) count = 0L; 80 put(signature, count+1); 81 return count+1; 82 } 83 } 84 85 final Counts failureCounts = new Counts(); 86 final static long maxFailures = 2; 87 88 final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\""); 89 final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}"); 90 91 boolean bug(String format, Object... args) { 92 String signature = String.format(format, args); 93 signature = hideBytes.matcher(signature).replaceAll("\"??\""); 94 signature = hideChars.matcher(signature).replaceAll("\\u????"); 95 failed++; 96 if (failureCounts.inc(signature) <= maxFailures) { 97 System.out.printf(format, args); 98 System.out.println(); 99 return true; 100 } 101 return false; 102 } 103 104 void summarize() { 105 for (String key : sort(failureCounts.keySet())) 106 System.out.printf("-----%n%s%nfailures=%d%n", 107 key, failureCounts.get(key)); 108 } 109 } 110 111 static final Reporter reporter = new Reporter(); 112 113 static class Result { 114 final int limit; 115 final int ipos; 116 final boolean direct; 117 final byte[] ia; 118 final char[] oa; 119 final CoderResult cr; 120 121 Result(ByteBuffer ib, CharBuffer ob, CoderResult cr) { 122 ipos = ib.position(); 123 ia = toArray(ib); 124 oa = toArray(ob); 125 direct = ib.isDirect(); 126 limit = ob.limit(); 127 this.cr = cr; 128 } 129 130 static byte[] toArray(ByteBuffer b) { 131 int pos = b.position(); 132 byte[] a = new byte[b.limit()]; 133 b.position(0); 134 b.get(a); 135 b.position(pos); 136 return a; 137 } 138 139 static char[] toArray(CharBuffer b) { 140 char[] a = new char[b.position()]; 141 b.position(0); 142 b.get(a); 143 return a; 144 } 145 146 static boolean eq(Result x, Result y) { 147 return x == y || 148 (x != null && y != null && 149 (Arrays.equals(x.oa, y.oa) && 150 x.ipos == y.ipos && 151 x.cr == y.cr)); 152 } 153 154 public String toString() { 155 return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s", 156 string(ia), ipos, ia.length, 157 cr, string(oa), oa.length, limit, 158 (direct ? " (direct)" : "")); 159 } 160 } 161 162 // legend: r=regular d=direct In=Input Ou=Output 163 static final int maxBufSize = 20; 164 static final ByteBuffer[] ribs = new ByteBuffer[maxBufSize]; 165 static final ByteBuffer[] dibs = new ByteBuffer[maxBufSize]; 166 167 static final CharBuffer[] robs = new CharBuffer[maxBufSize]; 168 static final CharBuffer[] dobs = new CharBuffer[maxBufSize]; 169 static { 170 for (int i = 0; i < maxBufSize; i++) { 171 ribs[i] = ByteBuffer.allocate(i); 172 dibs[i] = ByteBuffer.allocateDirect(i); 173 robs[i] = CharBuffer.allocate(i); 174 dobs[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer(); 175 } 176 } 177 178 static class CharsetTester { 179 private final Charset cs; 180 private static final long maxFailures = 5; 181 private long failures = 0; 182 // private static final long maxCharsetFailures = Long.MAX_VALUE; 183 private static final long maxCharsetFailures = 10000L; 184 private final long failed0 = failed; 185 186 CharsetTester(Charset cs) { 187 this.cs = cs; 188 } 189 190 static boolean bug(String format, Object... args) { 191 return reporter.bug(format, args); 192 } 193 194 Result recode(ByteBuffer ib, CharBuffer ob) { 195 try { 196 char canary = '\u4242'; 197 ib.clear(); // Prepare to read 198 ob.clear(); // Prepare to write 199 for (int i = 0; i < ob.limit(); i++) 200 ob.put(i, canary); 201 CharsetDecoder coder = cs.newDecoder(); 202 CoderResult cr = coder.decode(ib, ob, false); 203 equal(ib.limit(), ib.capacity()); 204 equal(ob.limit(), ob.capacity()); 205 Result r = new Result(ib, ob, cr); 206 if (cr.isError()) 207 check(cr.length() > 0); 208 if (cr.isOverflow() && ob.remaining() > 10) 209 bug("OVERFLOW, but there's lots of room: %s %s", 210 cs, r); 211 // if (cr.isOverflow() && ib.remaining() == 0) 212 // bug("OVERFLOW, yet remaining() == 0: %s %s", 213 // cs, r); 214 if (cr.isError() && ib.remaining() < cr.length()) 215 bug("remaining() < CoderResult.length(): %s %s", 216 cs, r); 217 // if (ib.position() == 0 && ob.position() > 0) 218 // reporter. bug("output only if input consumed: %s %s", 219 // cs, r); 220 // Should we warn if cr.isUnmappable() ?? 221 CoderResult cr2 = coder.decode(ib, ob, false); 222 if (ib.position() != r.ipos || 223 ob.position() != r.oa.length || 224 cr != cr2) 225 bug("Coding operation not idempotent: %s%n %s%n %s", 226 cs, r, new Result(ib, ob, cr2)); 227 if (ob.position() < ob.limit() && 228 ob.get(ob.position()) != canary) 229 bug("Buffer overrun: %s %s %s", 230 cs, r, ob.get(ob.position())); 231 return r; 232 } catch (Throwable t) { 233 if (bug("Unexpected exception: %s %s %s", 234 cs, t.getClass().getSimpleName(), 235 new Result(ib, ob, null))) 236 t.printStackTrace(); 237 return null; 238 } 239 } 240 241 Result recode2(byte[] ia, int n) { 242 int len = ia.length; 243 ByteBuffer rib = ByteBuffer.wrap(ia); 244 ByteBuffer dib = dibs[len]; 245 dib.clear(); dib.put(ia); dib.clear(); 246 CharBuffer rob = robs[n]; 247 CharBuffer dob = dobs[n]; 248 equal(rob.limit(), n); 249 equal(dob.limit(), n); 250 check(dib.isDirect()); 251 check(dob.isDirect()); 252 Result r1 = recode(rib, rob); 253 Result r2 = recode(dib, dob); 254 if (r1 != null && r2 != null && ! Result.eq(r1, r2)) 255 bug("Results differ for direct buffers: %s%n %s%n %s", 256 cs, r1, r2); 257 return r1; 258 } 259 260 Result test(byte[] ia) { 261 if (failed - failed0 >= maxCharsetFailures) 262 throw new TooManyFailures(); 263 264 Result roomy = recode2(ia, maxBufSize - 1); 265 if (roomy == null) return roomy; 266 int olen = roomy.oa.length; 267 if (olen > 0) { 268 if (roomy.ipos == roomy.ia.length) { 269 Result perfectFit = recode2(ia, olen); 270 if (! Result.eq(roomy, perfectFit)) 271 bug("Results differ: %s%n %s%n %s", 272 cs, roomy, perfectFit); 273 } 274 for (int i = 0; i < olen; i++) { 275 Result claustrophobic = recode2(ia, i); 276 if (claustrophobic == null) return roomy; 277 if (roomy.cr.isUnderflow() && 278 ! claustrophobic.cr.isOverflow()) 279 bug("Expected OVERFLOW: %s%n %s%n %s", 280 cs, roomy, claustrophobic); 281 } 282 } 283 return roomy; 284 } 285 286 void testExhaustively(byte[] prefix, int n) { 287 int len = prefix.length; 288 byte[] ia = Arrays.copyOf(prefix, len + 1); 289 for (int i = 0; i < 0x100; i++) { 290 ia[len] = (byte) i; 291 if (n == 1) 292 test(ia); 293 else 294 testExhaustively(ia, n - 1); 295 } 296 } 297 298 void testRandomly(byte[] prefix, int n) { 299 int len = prefix.length; 300 byte[] ia = Arrays.copyOf(prefix, len + n); 301 for (int i = 0; i < 5000; i++) { 302 for (int j = 0; j < n; j++) 303 ia[len + j] = randomByte(); 304 test(ia); 305 } 306 } 307 308 void testPrefix(byte[] prefix) { 309 if (prefix.length > 0) 310 System.out.printf("Testing prefix %s%n", string(prefix)); 311 312 test(prefix); 313 314 testExhaustively(prefix, 1); 315 testExhaustively(prefix, 2); 316 // Can you spare a week of CPU time? 317 // testExhaustively(cs, tester, prefix, 3); 318 319 testRandomly(prefix, 3); 320 testRandomly(prefix, 4); 321 } 322 } 323 324 private final static Random rnd = new Random(); 325 private static byte randomByte() { 326 return (byte) rnd.nextInt(0x100); 327 } 328 private static byte[] randomBytes(int len) { 329 byte[] a = new byte[len]; 330 for (int i = 0; i < len; i++) 331 a[i] = randomByte(); 332 return a; 333 } 334 335 private static final byte SS2 = (byte) 0x8e; 336 private static final byte SS3 = (byte) 0x8f; 337 private static final byte ESC = (byte) 0x1b; 338 private static final byte SO = (byte) 0x0e; 339 private static final byte SI = (byte) 0x0f; 340 341 private final static byte[][] stateChangers = { 342 {SS2}, {SS3}, {SO}, {SI} 343 }; 344 345 private final static byte[][]escapeSequences = { 346 {ESC, '(', 'B'}, 347 {ESC, '(', 'I'}, 348 {ESC, '(', 'J'}, 349 {ESC, '$', '@'}, 350 {ESC, '$', 'A'}, 351 {ESC, '$', ')', 'A'}, 352 {ESC, '$', ')', 'C'}, 353 {ESC, '$', ')', 'G'}, 354 {ESC, '$', '*', 'H'}, 355 {ESC, '$', '+', 'I'}, 356 {ESC, '$', 'B'}, 357 {ESC, 'N'}, 358 {ESC, 'O'}, 359 {ESC, '$', '(', 'D'}, 360 }; 361 362 private static boolean isStateChanger(Charset cs, byte[] ia) { 363 Result r = new CharsetTester(cs).recode2(ia, 9); 364 return r == null ? false : 365 (r.cr.isUnderflow() && 366 r.ipos == ia.length && 367 r.oa.length == 0); 368 } 369 370 private final static byte[][] incompletePrefixes = { 371 {ESC}, 372 {ESC, '('}, 373 {ESC, '$'}, 374 {ESC, '$', '(',}, 375 }; 376 377 private static boolean isIncompletePrefix(Charset cs, byte[] ia) { 378 Result r = new CharsetTester(cs).recode2(ia, 9); 379 return r == null ? false : 380 (r.cr.isUnderflow() && 381 r.ipos == 0 && 382 r.oa.length == 0); 383 } 384 385 private static void testCharset(Charset cs) throws Throwable { 386 final String csn = cs.name(); 387 388 if (isBroken(csn)) { 389 System.out.printf("Skipping possibly broken charset %s%n", csn); 390 return; 391 } 392 System.out.println(csn); 393 CharsetTester tester = new CharsetTester(cs); 394 395 tester.testPrefix(new byte[0]); 396 397 if (! csn.matches("(?:x-)?(?:UTF|JIS(?:_X)?0).*")) { 398 for (byte[] prefix : stateChangers) 399 if (isStateChanger(cs, prefix)) 400 tester.testPrefix(prefix); 401 402 for (byte[] prefix : incompletePrefixes) 403 if (isIncompletePrefix(cs, prefix)) 404 tester.testPrefix(prefix); 405 406 if (isIncompletePrefix(cs, new byte[] {ESC})) 407 for (byte[] prefix : escapeSequences) 408 if (isStateChanger(cs, prefix)) 409 tester.testPrefix(prefix); 410 } 411 } 412 413 private static void realMain(String[] args) { 414 for (Charset cs : sort(Charset.availableCharsets().values())) { 415 try { 416 testCharset(cs); 417 } catch (TooManyFailures e) { 418 System.out.printf("Too many failures for %s%n", cs); 419 } catch (Throwable t) { 420 unexpected(t); 421 } 422 } 423 reporter.summarize(); 424 } 425 426 //--------------------- Infrastructure --------------------------- 427 static volatile long passed = 0, failed = 0; 428 static void pass() {passed++;} 429 static void fail() {failed++; Thread.dumpStack();} 430 static void fail(String format, Object... args) { 431 System.out.println(String.format(format, args)); failed++;} 432 static void fail(String msg) {System.out.println(msg); fail();} 433 static void unexpected(Throwable t) {failed++; t.printStackTrace();} 434 static void check(boolean cond) {if (cond) pass(); else fail();} 435 static void equal(Object x, Object y) { 436 if (x == null ? y == null : x.equals(y)) pass(); 437 else fail(x + " not equal to " + y);} 438 static void equal(int x, int y) { 439 if (x == y) pass(); 440 else fail(x + " not equal to " + y);} 441 public static void main(String[] args) throws Throwable { 442 try {realMain(args);} catch (Throwable t) {unexpected(t);} 443 System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed); 444 if (failed > 0) throw new AssertionError("Some tests failed");} 445 }