1 /*
2 * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 /*
25 * @test
26 * @bug 6380723
27 * @summary Decode many byte sequences in many ways (use -Dseed=X to set PRNG seed)
28 * @library /test/lib
29 * @build jdk.test.lib.RandomFactory
30 * @run main/timeout=1800 FindDecoderBugs
31 * @author Martin Buchholz
32 * @key randomness
33 */
34
35 import java.util.*;
36 import java.util.regex.*;
37 import java.nio.*;
38 import java.nio.charset.*;
39 import jdk.test.lib.RandomFactory;
40
41 public class FindDecoderBugs {
42
43 static boolean isBroken(String csn) {
44 if (csn.equals("x-COMPOUND_TEXT")) return true;
45 return false;
46 }
47
48 static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) {
49 List<T> list = new ArrayList<T>(c);
50 Collections.sort(list);
51 return list;
52 }
53
54 static class TooManyFailures extends RuntimeException {
55 private static final long serialVersionUID = 0L;
56 }
57
58 static String string(byte[] a) {
59 final StringBuilder sb = new StringBuilder();
60 for (byte b : a) {
61 if (sb.length() != 0) sb.append(' ');
62 sb.append(String.format("%02x", b & 0xff));
63 }
64 return sb.toString();
65 }
66
67 static String string(char[] a) {
68 final StringBuilder sb = new StringBuilder();
69 for (char c : a) {
70 if (sb.length() != 0) sb.append(' ');
71 sb.append(String.format("\\u%04x", (int) c));
72 }
73 return sb.toString();
74 }
75
76 static class Reporter {
77 // Some machinery to make sure only a small number of errors
78 // that are "too similar" are reported.
79 static class Counts extends HashMap<String, Long> {
80 private static final long serialVersionUID = -1;
81 long inc(String signature) {
82 Long count = get(signature);
83 if (count == null) count = 0L;
84 put(signature, count+1);
85 return count+1;
86 }
87 }
88
89 final Counts failureCounts = new Counts();
90 final static long maxFailures = 2;
91
92 final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\"");
93 final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}");
94
95 boolean bug(String format, Object... args) {
96 String signature = String.format(format, args);
97 signature = hideBytes.matcher(signature).replaceAll("\"??\"");
98 signature = hideChars.matcher(signature).replaceAll("\\u????");
99 failed++;
100 if (failureCounts.inc(signature) <= maxFailures) {
101 System.out.printf(format, args);
102 System.out.println();
103 return true;
104 }
105 return false;
106 }
107
108 void summarize() {
109 for (String key : sort(failureCounts.keySet()))
110 System.out.printf("-----%n%s%nfailures=%d%n",
111 key, failureCounts.get(key));
112 }
113 }
114
115 static final Reporter reporter = new Reporter();
116
117 static class Result {
118 final int limit;
119 final int ipos;
120 final boolean direct;
121 final byte[] ia;
122 final char[] oa;
123 final CoderResult cr;
124
125 Result(ByteBuffer ib, CharBuffer ob, CoderResult cr) {
126 ipos = ib.position();
127 ia = toArray(ib);
128 oa = toArray(ob);
129 direct = ib.isDirect();
130 limit = ob.limit();
131 this.cr = cr;
132 }
133
134 static byte[] toArray(ByteBuffer b) {
135 int pos = b.position();
136 byte[] a = new byte[b.limit()];
137 b.position(0);
138 b.get(a);
139 b.position(pos);
140 return a;
141 }
142
143 static char[] toArray(CharBuffer b) {
144 char[] a = new char[b.position()];
145 b.position(0);
146 b.get(a);
147 return a;
148 }
149
150 static boolean eq(Result x, Result y) {
151 return x == y ||
152 (x != null && y != null &&
153 (Arrays.equals(x.oa, y.oa) &&
154 x.ipos == y.ipos &&
155 x.cr == y.cr));
156 }
157
158 public String toString() {
159 return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s",
160 string(ia), ipos, ia.length,
161 cr, string(oa), oa.length, limit,
162 (direct ? " (direct)" : ""));
163 }
164 }
165
166 // legend: r=regular d=direct In=Input Ou=Output
167 static final int maxBufSize = 20;
168 static final ByteBuffer[] ribs = new ByteBuffer[maxBufSize];
169 static final ByteBuffer[] dibs = new ByteBuffer[maxBufSize];
170
171 static final CharBuffer[] robs = new CharBuffer[maxBufSize];
172 static final CharBuffer[] dobs = new CharBuffer[maxBufSize];
173 static {
174 for (int i = 0; i < maxBufSize; i++) {
175 ribs[i] = ByteBuffer.allocate(i);
176 dibs[i] = ByteBuffer.allocateDirect(i);
177 robs[i] = CharBuffer.allocate(i);
178 dobs[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer();
179 }
180 }
181
182 static class CharsetTester {
183 private final Charset cs;
184 private static final long maxFailures = 5;
185 private long failures = 0;
186 // private static final long maxCharsetFailures = Long.MAX_VALUE;
187 private static final long maxCharsetFailures = 10000L;
188 private final long failed0 = failed;
189
190 CharsetTester(Charset cs) {
191 this.cs = cs;
192 }
193
194 static boolean bug(String format, Object... args) {
195 return reporter.bug(format, args);
196 }
197
198 Result recode(ByteBuffer ib, CharBuffer ob) {
199 try {
200 char canary = '\u4242';
201 ib.clear(); // Prepare to read
202 ob.clear(); // Prepare to write
203 for (int i = 0; i < ob.limit(); i++)
204 ob.put(i, canary);
205 CharsetDecoder coder = cs.newDecoder();
206 CoderResult cr = coder.decode(ib, ob, false);
207 equal(ib.limit(), ib.capacity());
208 equal(ob.limit(), ob.capacity());
209 Result r = new Result(ib, ob, cr);
210 if (cr.isError())
211 check(cr.length() > 0);
212 if (cr.isOverflow() && ob.remaining() > 10)
213 bug("OVERFLOW, but there's lots of room: %s %s",
214 cs, r);
215 // if (cr.isOverflow() && ib.remaining() == 0)
216 // bug("OVERFLOW, yet remaining() == 0: %s %s",
217 // cs, r);
218 if (cr.isError() && ib.remaining() < cr.length())
219 bug("remaining() < CoderResult.length(): %s %s",
220 cs, r);
221 // if (ib.position() == 0 && ob.position() > 0)
222 // reporter. bug("output only if input consumed: %s %s",
223 // cs, r);
224 // Should we warn if cr.isUnmappable() ??
225 CoderResult cr2 = coder.decode(ib, ob, false);
226 if (ib.position() != r.ipos ||
227 ob.position() != r.oa.length ||
228 cr != cr2)
229 bug("Coding operation not idempotent: %s%n %s%n %s",
230 cs, r, new Result(ib, ob, cr2));
231 if (ob.position() < ob.limit() &&
232 ob.get(ob.position()) != canary)
233 bug("Buffer overrun: %s %s %s",
234 cs, r, ob.get(ob.position()));
235 return r;
236 } catch (Throwable t) {
237 if (bug("Unexpected exception: %s %s %s",
238 cs, t.getClass().getSimpleName(),
239 new Result(ib, ob, null)))
240 t.printStackTrace();
241 return null;
242 }
243 }
244
245 Result recode2(byte[] ia, int n) {
246 int len = ia.length;
247 ByteBuffer rib = ByteBuffer.wrap(ia);
248 ByteBuffer dib = dibs[len];
249 dib.clear(); dib.put(ia); dib.clear();
250 CharBuffer rob = robs[n];
251 CharBuffer dob = dobs[n];
252 equal(rob.limit(), n);
253 equal(dob.limit(), n);
254 check(dib.isDirect());
255 check(dob.isDirect());
256 Result r1 = recode(rib, rob);
257 Result r2 = recode(dib, dob);
258 if (r1 != null && r2 != null && ! Result.eq(r1, r2))
259 bug("Results differ for direct buffers: %s%n %s%n %s",
260 cs, r1, r2);
261 return r1;
262 }
263
264 Result test(byte[] ia) {
265 if (failed - failed0 >= maxCharsetFailures)
266 throw new TooManyFailures();
267
268 Result roomy = recode2(ia, maxBufSize - 1);
269 if (roomy == null) return roomy;
270 int olen = roomy.oa.length;
271 if (olen > 0) {
272 if (roomy.ipos == roomy.ia.length) {
273 Result perfectFit = recode2(ia, olen);
274 if (! Result.eq(roomy, perfectFit))
275 bug("Results differ: %s%n %s%n %s",
276 cs, roomy, perfectFit);
277 }
278 for (int i = 0; i < olen; i++) {
279 Result claustrophobic = recode2(ia, i);
280 if (claustrophobic == null) return roomy;
281 if (roomy.cr.isUnderflow() &&
282 ! claustrophobic.cr.isOverflow())
283 bug("Expected OVERFLOW: %s%n %s%n %s",
284 cs, roomy, claustrophobic);
285 }
286 }
287 return roomy;
288 }
289
290 void testExhaustively(byte[] prefix, int n) {
291 int len = prefix.length;
292 byte[] ia = Arrays.copyOf(prefix, len + 1);
293 for (int i = 0; i < 0x100; i++) {
294 ia[len] = (byte) i;
295 if (n == 1)
296 test(ia);
297 else
298 testExhaustively(ia, n - 1);
299 }
300 }
301
302 void testRandomly(byte[] prefix, int n) {
303 int len = prefix.length;
304 byte[] ia = Arrays.copyOf(prefix, len + n);
305 for (int i = 0; i < 5000; i++) {
306 for (int j = 0; j < n; j++)
307 ia[len + j] = randomByte();
308 test(ia);
309 }
310 }
311
312 void testPrefix(byte[] prefix) {
313 if (prefix.length > 0)
314 System.out.printf("Testing prefix %s%n", string(prefix));
315
316 test(prefix);
317
318 testExhaustively(prefix, 1);
319 testExhaustively(prefix, 2);
320 // Can you spare a week of CPU time?
321 // testExhaustively(cs, tester, prefix, 3);
322
323 testRandomly(prefix, 3);
324 testRandomly(prefix, 4);
325 }
326 }
327
328 private final static Random rnd = RandomFactory.getRandom();
329 private static byte randomByte() {
330 return (byte) rnd.nextInt(0x100);
331 }
332 private static byte[] randomBytes(int len) {
333 byte[] a = new byte[len];
334 for (int i = 0; i < len; i++)
335 a[i] = randomByte();
336 return a;
337 }
338
339 private static final byte SS2 = (byte) 0x8e;
340 private static final byte SS3 = (byte) 0x8f;
341 private static final byte ESC = (byte) 0x1b;
342 private static final byte SO = (byte) 0x0e;
343 private static final byte SI = (byte) 0x0f;
344
345 private final static byte[][] stateChangers = {
346 {SS2}, {SS3}, {SO}, {SI}
347 };
348
349 private final static byte[][]escapeSequences = {
350 {ESC, '(', 'B'},
351 {ESC, '(', 'I'},
352 {ESC, '(', 'J'},
353 {ESC, '$', '@'},
354 {ESC, '$', 'A'},
355 {ESC, '$', ')', 'A'},
356 {ESC, '$', ')', 'C'},
357 {ESC, '$', ')', 'G'},
358 {ESC, '$', '*', 'H'},
359 {ESC, '$', '+', 'I'},
360 {ESC, '$', 'B'},
361 {ESC, 'N'},
362 {ESC, 'O'},
363 {ESC, '$', '(', 'D'},
364 };
365
366 private static boolean isStateChanger(Charset cs, byte[] ia) {
367 Result r = new CharsetTester(cs).recode2(ia, 9);
368 return r == null ? false :
369 (r.cr.isUnderflow() &&
370 r.ipos == ia.length &&
371 r.oa.length == 0);
372 }
373
374 private final static byte[][] incompletePrefixes = {
375 {ESC},
376 {ESC, '('},
377 {ESC, '$'},
378 {ESC, '$', '(',},
379 };
380
381 private static boolean isIncompletePrefix(Charset cs, byte[] ia) {
382 Result r = new CharsetTester(cs).recode2(ia, 9);
383 return r == null ? false :
384 (r.cr.isUnderflow() &&
385 r.ipos == 0 &&
386 r.oa.length == 0);
387 }
388
389 private static void testCharset(Charset cs) throws Throwable {
390 final String csn = cs.name();
391
392 if (isBroken(csn)) {
393 System.out.printf("Skipping possibly broken charset %s%n", csn);
394 return;
395 }
396 System.out.println(csn);
397 CharsetTester tester = new CharsetTester(cs);
398
399 tester.testPrefix(new byte[0]);
400
401 if (! csn.matches("(?:x-)?(?:UTF|JIS(?:_X)?0).*")) {
402 for (byte[] prefix : stateChangers)
403 if (isStateChanger(cs, prefix))
404 tester.testPrefix(prefix);
405
406 for (byte[] prefix : incompletePrefixes)
407 if (isIncompletePrefix(cs, prefix))
408 tester.testPrefix(prefix);
409
410 if (isIncompletePrefix(cs, new byte[] {ESC}))
411 for (byte[] prefix : escapeSequences)
412 if (isStateChanger(cs, prefix))
413 tester.testPrefix(prefix);
414 }
415 }
416
417 private static void realMain(String[] args) {
418 for (Charset cs : sort(Charset.availableCharsets().values())) {
419 try {
420 testCharset(cs);
421 } catch (TooManyFailures e) {
422 System.out.printf("Too many failures for %s%n", cs);
423 } catch (Throwable t) {
424 unexpected(t);
425 }
426 }
427 reporter.summarize();
428 }
429
430 //--------------------- Infrastructure ---------------------------
431 static volatile long passed = 0, failed = 0;
432 static void pass() {passed++;}
433 static void fail() {failed++; Thread.dumpStack();}
434 static void fail(String format, Object... args) {
435 System.out.println(String.format(format, args)); failed++;}
436 static void fail(String msg) {System.out.println(msg); fail();}
437 static void unexpected(Throwable t) {failed++; t.printStackTrace();}
438 static void check(boolean cond) {if (cond) pass(); else fail();}
439 static void equal(Object x, Object y) {
440 if (x == null ? y == null : x.equals(y)) pass();
441 else fail(x + " not equal to " + y);}
442 static void equal(int x, int y) {
443 if (x == y) pass();
444 else fail(x + " not equal to " + y);}
445 public static void main(String[] args) throws Throwable {
446 try {realMain(args);} catch (Throwable t) {unexpected(t);}
447 System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed);
448 if (failed > 0) throw new AssertionError("Some tests failed");}
449 }