1 /*
2 * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 /* @test
25 * @bug 6636323 6636319 7040220 7096080 7183053 8080248 8054307 8021560
26 * @summary Test if StringCoding and NIO result have the same de/encoding result
27 * @modules java.base/sun.nio.cs
28 * @run main/othervm/timeout=2000 TestStringCoding
29 * @key randomness
30 */
31
32 import java.util.*;
33 import java.nio.*;
34 import java.nio.charset.*;
35
36 public class TestStringCoding {
37
38 private static Random r = new Random();
39
40 public static void main(String[] args) throws Throwable {
41
42 // full bmp first
43 char[] bmp = new char[0x10000];
44 for (int i = 0; i < 0x10000; i++) {
45 bmp[i] = (char)i;
46 }
47 char[] latin = Arrays.copyOf(bmp, 0x100);
48 char[] ascii = Arrays.copyOf(bmp, 0x80);
49
50 byte[] latinBA = new byte[0x100];
51 for (int i = 0; i < 0x100; i++) {
52 latinBA[i] = (byte)i;
53 }
54 byte[] asciiBA = Arrays.copyOf(latinBA, 0x80);
55
56 for (Boolean hasSM: new boolean[] { false, true }) {
57 if (hasSM) {
58 System.setSecurityManager(new PermissiveSecurityManger());
59 }
60 for (Charset cs: Charset.availableCharsets().values()) {
61
62 if (!cs.name().contains("Big5-HKSCS"))
63 continue;
64
65 if ("ISO-2022-CN".equals(cs.name()) ||
66 "x-COMPOUND_TEXT".equals(cs.name()) ||
67 "x-JISAutoDetect".equals(cs.name()))
68 continue;
69
70 System.out.printf("Testing(sm=%b) " + cs.name() + "....", hasSM);
71
72 testNewString(cs, testGetBytes(cs, new String(bmp)));
73 testNewString(cs, testGetBytes(cs, new String(latin)));
74 testNewString(cs, testGetBytes(cs, new String(ascii)));
75 testGetBytes(cs, testNewString(cs, latinBA));
76 testGetBytes(cs, testNewString(cs, asciiBA));
77
78 // "randomed" sizes
79 Random rnd = new Random();
80 for (int i = 0; i < 10; i++) {
81 //System.out.printf(" blen=%d, clen=%d%n", blen, clen);
82 char[] bmp0 = Arrays.copyOf(bmp, rnd.nextInt(0x10000));
83 testNewString(cs, testGetBytes(cs, new String(bmp0)));
84 //add a pair of surrogates
85 int pos = bmp0.length / 2;
86 if ((pos + 1) < bmp0.length) {
87 bmp0[pos] = '\uD800';
88 bmp0[pos+1] = '\uDC00';
89 }
90 testNewString(cs, testGetBytes(cs, new String(bmp0)));
91
92 char[] latin0 = Arrays.copyOf(latin, rnd.nextInt(0x100));
93 char[] ascii0 = Arrays.copyOf(ascii, rnd.nextInt(0x80));
94 byte[] latinBA0 = Arrays.copyOf(latinBA, rnd.nextInt(0x100));
95 byte[] asciiBA0 = Arrays.copyOf(asciiBA, rnd.nextInt(0x80));
96 testNewString(cs, testGetBytes(cs, new String(latin0)));
97 testNewString(cs, testGetBytes(cs, new String(ascii0)));
98 testGetBytes(cs, testNewString(cs, latinBA0));
99 testGetBytes(cs, testNewString(cs, asciiBA0));
100 }
101 testSurrogates(cs);
102 testMixed(cs);
103 System.out.println("done!");
104 }
105 }
106 }
107
108 static void testMixed(Charset cs) throws Throwable {
109 CharsetDecoder dec = cs.newDecoder()
110 .onMalformedInput(CodingErrorAction.REPLACE)
111 .onUnmappableCharacter(CodingErrorAction.REPLACE);
112 CharsetEncoder enc = cs.newEncoder()
113 .onMalformedInput(CodingErrorAction.REPLACE)
114 .onUnmappableCharacter(CodingErrorAction.REPLACE);
115 List<Integer> cps = new ArrayList<>(0x10000);
116 int off = 0;
117 int cp = 0;
118 while (cp < 0x10000) {
119 if (enc.canEncode((char)cp)) {
120 cps.add(cp);
121 }
122 cp++;
123 }
124 Collections.shuffle(cps);
125 char[] bmpCA = new char[cps.size()];
126 for (int i = 0; i < cps.size(); i++)
127 bmpCA[i] = (char)(int)cps.get(i);
128 String bmpStr = new String(bmpCA);
129 // test getBytes()
130 byte[] bmpBA = testGetBytes(cs, bmpStr);
131 // test new String()
132 testNewString(cs, bmpBA);
133 }
134
135 static byte[] getBytes(CharsetEncoder enc, String str) throws Throwable {
136 ByteBuffer bf = enc.reset().encode(CharBuffer.wrap(str.toCharArray()));
137 byte[] ba = new byte[bf.limit()];
138 bf.get(ba, 0, ba.length);
139 return ba;
140 }
141
142 static void assertEquals(int len1, int len2, String msg) {
143 if (len1 != len2)
144 throw new RuntimeException("FAILED: " + msg);
145 }
146
147 static void assertEquals(byte[] a, int afrom, int ato,
148 byte[] b, int bfrom, int bto,
149 String msg) {
150 if (!Arrays.equals(a, afrom, ato, b, bfrom, bto)) {
151 throw new RuntimeException("FAILED: " + msg);
152 }
153 }
154
155 static void assertEquals(ByteBuffer bb, int pos, byte[] b, int from, int to,
156 byte[] buf, String msg) {
157 bb.flip().position(pos);
158 int len = bb.remaining();
159 bb.get(buf, 0, len);
160 if (!Arrays.equals(buf, 0, len, b, from, to)) {
161 throw new RuntimeException("FAILED: " + msg);
162 }
163 }
164
165 static byte[] testGetBytes(Charset cs, String str) throws Throwable {
166 CharsetEncoder enc = cs.newEncoder()
167 .onMalformedInput(CodingErrorAction.REPLACE)
168 .onUnmappableCharacter(CodingErrorAction.REPLACE)
169 .reset();
170
171 //getBytes(csn);
172 String csn = cs.name();
173 byte[] baSC = str.getBytes(csn);
174 byte[] baNIO = getBytes(enc, str);
175 assertEquals(baSC, 0, baSC.length, baNIO, 0, baNIO.length, "getBytes(csn)");
176
177 //getBytes(cs);
178 baSC = str.getBytes(cs);
179 assertEquals(baSC, 0, baSC.length, baNIO, 0, baNIO.length, "getBytes(cs)");
180
181 byte[] buf = new byte[baNIO.length + 100];
182 int shortenLen = Math.max(0, baNIO.length - 10);
183 int slen = str.length();
184
185 // getBytes(int, int ByteBuffer, cs);
186 ByteBuffer bb = ByteBuffer.wrap(buf);
187 int n = str.getBytes(0, slen, bb, cs);
188 int bn = bb.position();
189 assertEquals(n, slen, "getBytes(bb cs)");
190 assertEquals(buf, 0, bn, baNIO, 0, baNIO.length, "getBytes(bb, cs)");
191
192 bb.clear().position(3);
193 n = str.getBytes(0, slen, bb, cs);
194 bn = bb.position();
195 assertEquals(n, slen, "getBytes(bb/3, cs)");
196 assertEquals(buf, 3, bn, baNIO, 0, baNIO.length, "getBytes(bb/3, cs)");
197
198 bb.clear().limit(shortenLen);
199 n = str.getBytes(0, slen, bb, cs);
200 bn = bb.position();
201 assertEquals(buf, 0, bn, baNIO, 0, bn, "getBytes(bb, cs)/shortenLen");
202
203 System.out.printf ("testGetBytes.loop: str.len=%d, buf.len=%d, baNIO.len=%d%n",
204 slen, buf.length, baNIO.length);
205
206 int off = 0;
207 int m = 0;
208 bb.clear();
209 while (off < slen) {
210 int nn = slen - off;
211 if (m++ < 10 && nn > 10) {
212 nn = r.nextInt(nn);
213 }
214 n = str.getBytes(off, off + nn, bb, cs);
215 off += n;
216 System.out.printf (" ---> off=%d, slen=%d, bb.rem=%d, n=%d, nn=%d m=%d%n", off, slen, bb.remaining(), n, nn, m);
217 }
218
219 bb.flip();
220 // assertEquals(buf, 0, bn, baNIO, 0, bn, "getBytes(bb, cs)/loop");
221
222 if (!new String(bb, cs).equals(new String(baNIO, cs))) {
223 bb.rewind();
224 String s1 = new String(bb, cs);
225 String s2 = new String(baNIO, cs);
226
227 System.out.printf(" bb.str.len=%d, nio.str.len=%d%n", s1.length(), s2.length());
228 for (int i = 0 ;i < s1.length(); i++) {
229 if (s1.charAt(i) != s2.charAt(i)) {
230 System.out.printf(" [%d] %x %x%n", i, s1.charAt(i) & 0xffff, s2.charAt(i) & 0xffff);
231 }
232 }
233
234
235 throw new RuntimeException("getBytes(bb, cs)/loop");
236 //new RuntimeException("getBytes(bb, cs)/loop").printStackTrace();
237 }
238
239 // getBytes(int, int, ByteBuffer/direct, cs);
240 bb = ByteBuffer.allocateDirect(buf.length);
241 n = str.getBytes(0, slen, bb, cs);
242 assertEquals(n, slen, "getBytes(bb cs)");
243 assertEquals(bb, 0, baNIO, 0, baNIO.length, buf, "getBytes(bb, cs)");
244
245 bb.clear().position(3);
246 n = str.getBytes(0, slen, bb, cs);
247 assertEquals(n, slen, "getBytes(bb, cs)");
248 assertEquals(bb, 3, baNIO, 0, baNIO.length, buf, "getBytes(bb/3, cs)");
249
250 bb.clear().limit(shortenLen);
251 n = str.getBytes(0, slen, bb, cs);
252 bn = bb.position();
253 assertEquals(bb, 0, baNIO, 0, bn, buf, "getBytes(bb, cs)/shortenLen");
254
255 return baSC;
256 }
257
258 static String testNewString(Charset cs, byte[] ba) throws Throwable {
259 CharsetDecoder dec = cs.newDecoder()
260 .onMalformedInput(CodingErrorAction.REPLACE)
261 .onUnmappableCharacter(CodingErrorAction.REPLACE);
262
263 String csn = cs.name();
264
265 //new String(csn);
266 String strSC = new String(ba, cs.name());
267 String strNIO = dec.reset().decode(ByteBuffer.wrap(ba)).toString();
268 if(!strNIO.equals(strSC)) {
269 throw new RuntimeException("new String(csn) failed -> " + csn);
270 }
271 //new String(cs);
272 strSC = new String(ba, cs);
273 if (!strNIO.equals(strSC)) {
274 throw new RuntimeException("new String(cs) failed -> " + csn);
275 }
276
277 //new String(bytebuffer, cs);
278 ByteBuffer bb = ByteBuffer.allocate(ba.length + 10);
279 bb.put(ba);
280 bb.flip();
281 if (!strNIO.equals(new String(bb, cs))) {
282 throw new RuntimeException("new String(bb, cs) failed -> " + csn);
283 }
284
285 bb.clear().position(3).put(ba).flip();
286 if (!strNIO.equals(new String(bb.position(3), cs))) {
287 throw new RuntimeException("new String(cs)/pos=3 failed -> " + csn);
288 }
289
290 //new String(bytebuffer/direct, cs);
291 bb = ByteBuffer.allocateDirect(ba.length + 10);
292 bb.put(ba);
293 bb.flip();
294 if (!strNIO.equals(new String(bb, cs))) {
295 throw new RuntimeException("new String(cs)/bmp failed -> " + csn);
296 }
297
298 bb.clear().position(3).put(ba).flip();
299 if (!strNIO.equals(new String(bb.position(3), cs))) {
300 throw new RuntimeException("new String(cs)/pos=3 failed -> " + csn);
301 }
302
303 return strSC;
304 }
305
306 static void testSurrogates(Charset cs) throws Throwable {
307 //encode unmappable surrogates
308 CharsetEncoder enc = cs.newEncoder()
309 .onMalformedInput(CodingErrorAction.REPLACE)
310 .onUnmappableCharacter(CodingErrorAction.REPLACE);
311 if (enc instanceof sun.nio.cs.ArrayEncoder &&
312 cs.contains(Charset.forName("ASCII"))) {
313 if (cs.name().equals("UTF-8") || // utf8 handles surrogates
314 cs.name().equals("CESU-8")) // utf8 handles surrogates
315 return;
316 enc.replaceWith(new byte[] { (byte)'A'});
317 sun.nio.cs.ArrayEncoder cae = (sun.nio.cs.ArrayEncoder)enc;
318
319 String str = "ab\uD800\uDC00\uD800\uDC00cd";
320 byte[] ba = new byte[str.length() - 2];
321 int n = cae.encode(str.toCharArray(), 0, str.length(), ba);
322 if (n != 6 || !"abAAcd".equals(new String(ba, cs.name())))
323 throw new RuntimeException("encode1(surrogates) failed -> "
324 + cs.name());
325
326 ba = new byte[str.length()];
327 n = cae.encode(str.toCharArray(), 0, str.length(), ba);
328 if (n != 6 || !"abAAcd".equals(new String(ba, 0, n,
329 cs.name())))
330 throw new RuntimeException("encode2(surrogates) failed -> "
331 + cs.name());
332 str = "ab\uD800B\uDC00Bcd";
333 ba = new byte[str.length()];
334 n = cae.encode(str.toCharArray(), 0, str.length(), ba);
335 if (n != 8 || !"abABABcd".equals(new String(ba, 0, n,
336 cs.name())))
337 throw new RuntimeException("encode3(surrogates) failed -> "
338 + cs.name());
339 /* sun.nio.cs.ArrayDeEncoder works on the assumption that the
340 invoker (StringCoder) allocates enough output buf, utf8
341 and double-byte coder does not check the output buffer limit.
342 ba = new byte[str.length() - 1];
343 n = cae.encode(str.toCharArray(), 0, str.length(), ba);
344 if (n != 7 || !"abABABc".equals(new String(ba, 0, n, cs.name()))) {
345 throw new RuntimeException("encode4(surrogates) failed -> "
346 + cs.name());
347 }
348 */
349 }
350
351 //encode mappable surrogates for hkscs
352 if (cs.name().equals("Big5-HKSCS") || cs.name().equals("x-MS950-HKSCS")) {
353 String str = "ab\uD840\uDD0Ccd";
354 byte[] expected = new byte[] {(byte)'a', (byte)'b',
355 (byte)0x88, (byte)0x45, (byte)'c', (byte)'d' };
356 if (!Arrays.equals(str.getBytes(cs.name()), expected) ||
357 !Arrays.equals(str.getBytes(cs), expected)) {
358 throw new RuntimeException("encode(surrogates) failed -> "
359 + cs.name());
360 }
361 }
362 }
363
364 static class PermissiveSecurityManger extends SecurityManager {
365 @Override public void checkPermission(java.security.Permission p) {}
366 }
367 }