1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 /**
27 *
28 * @author Alan Liu
29 * @author John O'Conner
30 */
31
32 import java.io.*;
33
34
35 /**
36 * This class either loads or dumps the character properties of all Unicode
37 * characters out to a file. When loading, it compares the loaded data with
38 * that obtained through the java.lang.Character API. This allows detection of
39 * changes to the character properties between versions of the Java VM. A
40 * typical usage would be to dump the properties under an early VM, and load
41 * them under a later VM.
42 *
43 * Also: Check the current VM's character properties against those in a
44 * Unicode database. The database should be of the format
45 * available on ftp.unicode.org/Public/UNIDATA.
46 *
47 */
48 public class CharCheck {
49 static int differences = 0;
50
51 public static void main(String args[]) throws Exception {
52
53 if (args.length != 2 && args.length != 3) usage();
54 if (args[0].equals("dump"))
55 dump(Integer.parseInt(args[1], 16), new ObjectOutputStream(new FileOutputStream(args[2])));
56 else if (args[0].equals("load"))
57 load(Integer.parseInt(args[1], 16), new ObjectInputStream(new FileInputStream(args[2])));
58 else if (args[0].equals("check"))
59 check(Integer.parseInt(args[1], 16), new File(args[2]));
60 else if (args[0].equals("char"))
61 showChar(Integer.parseInt(args[1],16));
62 else if (args[0].equals("fchar"))
63 showFileChar(args[1], Integer.parseInt(args[2],16));
64 else usage();
65 if (differences != 0) {
66 throw new RuntimeException("There are differences between Character properties and the specification.");
67 }
68 }
69
70 static void usage() {
71 System.err.println("Usage: java CharCheck <command>");
72 System.err.println("where <command> is one of the following:");
73 System.err.println("dump <plane> <file> - dumps the character properties of the given plane,");
74 System.err.println(" read from the current VM, to the given file.");
75 System.err.println("load <plane> <file> - loads the character properties from the given");
76 System.err.println(" file and compares them to those of the given character plane");
77 System.err.println(" in the current VM.");
78 System.err.println("check <plane> <file> - compare the current VM's character properties");
79 System.err.println(" in the given plane to those listed in the given file, ");
80 System.err.println(" which should be in the format available on ");
81 System.err.println(" ftp.unicode.org/Public/2.0-Update.");
82 System.err.println("char <code> - show current VM properties of the given Unicode char.");
83 System.err.println("fchar <file> <code> - show file properties of the given Unicode char.");
84 System.exit(0);
85 }
86
87 static String getTypeName(int type) {
88 return (type >= 0 && type < UnicodeSpec.generalCategoryList.length) ?
89 (UnicodeSpec.generalCategoryList[type][UnicodeSpec.LONG] + '(' + type + ')') :
90 ("<Illegal type value " + type + ">");
91 }
92
93 static int check(int plane, File specFile) throws Exception {
94
95 String version = System.getProperty("java.version");
96 System.out.println("Current VM version " + version);
97 int rangeLimit = (plane << 16) | 0xFFFF;
98 String record;
99 UnicodeSpec[] spec = UnicodeSpec.readSpecFile(specFile, plane);
100 int rangeStart = 0x0000;
101 boolean isRange = false;
102
103 lastCheck = (plane << 16) - 1;
104
105 for (int currentSpec = 0; currentSpec < spec.length; currentSpec++) {
106 int c = spec[currentSpec].getCodePoint();
107 if (isRange) {
108 // Must see end of range now
109 if (spec[currentSpec].getName().endsWith("Last>")) {
110 for (int d=rangeStart; d<=c; d++) {
111 checkOneChar(d, spec[currentSpec]);
112 }
113 }
114 else {
115 // No good -- First without Last
116 System.out.println("BAD FILE: First without last at '" + escape(rangeStart) + "'");
117 }
118 isRange = false;
119 }
120 else {
121 // Look for a First, Last pair: This is a pair of entries like the following:
122 // 4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
123 // 9FA5;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
124 if (spec[currentSpec].getName().endsWith("First>")) {
125 rangeStart = c;
126 isRange = true;
127 }
128 else {
129 checkOneChar(c, spec[currentSpec]);
130 }
131 }
132 }
133
134 // Check undefined chars at the end of the range
135
136 while (lastCheck < rangeLimit) checkOneCharDefined(++lastCheck, "?", false);
137
138 System.out.println("Total differences: "+differences);
139 return differences;
140 }
141
142 static int lastCheck = -1;
143
144 static final void checkOneCharDefined(int c, String name, boolean fileDefined) {
145 if (Character.isDefined(c) != fileDefined)
146 showDifference(c, name, "isDefined", ""+(!fileDefined), ""+fileDefined);
147 }
148
149 // In GenerateCharacter, the following ranges are handled specially.
150 // Each is the start of a 26-character range with values 10..35.
151 static final char NUMERIC_EXCEPTION[] = { '\u0041', '\u0061', '\uFF21', '\uFF41' };
152
153 static void checkOneChar(int c, UnicodeSpec charSpec) {
154 // Handle intervening ranges -- we assume that we will be called in monotonically
155 // increasing order. If the last char we checked is more than one before this
156 // char, then check the intervening range -- it should all be undefined.
157 int lowerLimit = (c & 0xFF0000);
158 if (lastCheck >= lowerLimit && (lastCheck+1) != c) {
159 for (int i=lastCheck+1; i<c; ++i)
160 checkOneCharDefined(i, "?", false);
161 }
162
163 lastCheck = c;
164
165 // isDefined should be true
166 checkOneCharDefined(c, charSpec.getName(), true);
167
168 // Check lower, upper, and titlecase conversion
169 int upper = Character.toUpperCase(c);
170 int lower = Character.toLowerCase(c);
171 int title = Character.toTitleCase(c);
172 int upperDB = charSpec.hasUpperMap() ? charSpec.getUpperMap() : c;
173 int lowerDB = charSpec.hasLowerMap() ? charSpec.getLowerMap() : c;
174 int titleDB = charSpec.hasTitleMap() ? charSpec.getTitleMap() : c;
175 if (upper != upperDB) showDifference(c, charSpec.getName(), "upper", hex6(upper), hex6(upperDB));
176 if (lower != lowerDB) showDifference(c, charSpec.getName(), "lower", hex6(lower), hex6(lowerDB));
177 if (title != titleDB) showDifference(c, charSpec.getName(), "title", hex6(title), hex6(titleDB));
178
179 // Check the character general category (type)
180 int type = Character.getType(c);
181 int typeDB = charSpec.getGeneralCategory();
182 if (type != typeDB) {
183 showDifference(c, charSpec.getName(), "type",
184 UnicodeSpec.generalCategoryList[type][UnicodeSpec.SHORT],
185 UnicodeSpec.generalCategoryList[typeDB][UnicodeSpec.SHORT]);
186 }
187
188 // Check the mirrored property
189 boolean isMirrored = Character.isMirrored(c);
190 boolean isMirroredDB = charSpec.isMirrored();
191 if (isMirrored != isMirroredDB) {
192 showDifference(c, charSpec.getName(), "isMirrored", ""+isMirrored, ""+isMirroredDB);
193 }
194
195 // Check the directionality property
196 byte directionality = Character.getDirectionality(c);
197 byte directionalityDB = charSpec.getBidiCategory();
198 if (directionality != directionalityDB) {
199 showDifference(c, charSpec.getName(), "directionality", ""+directionality, ""+directionalityDB);
200 }
201
202 // Check the decimal digit property
203 int decimalDigit = Character.digit(c, 10);
204 int decimalDigitDB = -1;
205 if (charSpec.getGeneralCategory() == UnicodeSpec.DECIMAL_DIGIT_NUMBER) {
206 decimalDigitDB = charSpec.getDecimalValue();
207 }
208 if (decimalDigit != decimalDigitDB)
209 showDifference(c, charSpec.getName(), "decimal digit", ""+decimalDigit, ""+decimalDigitDB);
210
211 // Check the numeric property
212 int numericValue = Character.getNumericValue(c);
213 int numericValueDB;
214 if (charSpec.getNumericValue().length() == 0) {
215 numericValueDB = -1;
216 // Handle exceptions where Character deviates from the UCS spec
217 for (int k=0; k<NUMERIC_EXCEPTION.length; ++k) {
218 if (c >= NUMERIC_EXCEPTION[k] && c < (char)(NUMERIC_EXCEPTION[k]+26)) {
219 numericValueDB = c - NUMERIC_EXCEPTION[k] + 10;
220 break;
221 }
222 }
223 }
224 else {
225 String strValue = charSpec.getNumericValue();
226 int parsedNumericValue;
227 if (strValue.equals("10000000000")
228 || strValue.equals("1000000000000")) {
229 System.out.println("Skipping strValue: " + strValue
230 + " for " + charSpec.getName()
231 + "(0x" + Integer.toHexString(c) + ")");
232 parsedNumericValue = -2;
233 } else {
234 parsedNumericValue = strValue.indexOf('/') < 0 ?
235 Integer.parseInt(strValue) : -2;
236 }
237 numericValueDB = parsedNumericValue < 0 ? -2 : parsedNumericValue;
238 }
239 if (numericValue != numericValueDB)
240 showDifference(c, charSpec.getName(), "numeric value", ""+numericValue, ""+numericValueDB);
241 }
242
243 static void showDifference(int c, String name, String property, String vmValue, String dbValue) {
244 System.out.println(escape("Mismatch at '" + hex6(c) + "' (" + name+ "): " +
245 property + "=" + vmValue + ", db=" + dbValue));
246 ++differences;
247 }
248
249 /**
250 * Given a record containing ';'-separated fields, return the fieldno-th
251 * field. The first field is field 0.
252 */
253 static String getField(String record, int fieldno) {
254 int i=0;
255 int j=record.indexOf(';');
256 while (fieldno > 0) {
257 i=j+1;
258 j=record.indexOf(';', i);
259 }
260 return record.substring(i, j);
261 }
262
263 static final int FIELD_COUNT = 15;
264
265 /**
266 * Given a record containing ';'-separated fields, return an array of
267 * the fields. It is assumed that there are FIELD_COUNT fields per record.
268 */
269 static void getFields(String record, String[] fields) {
270 int i=0;
271 int j=record.indexOf(';');
272 fields[0] = record.substring(i, j);
273 for (int n=1; n<FIELD_COUNT; ++n) {
274 i=j+1;
275 j=record.indexOf(';', i);
276 fields[n] = (j<0) ? record.substring(i) : record.substring(i, j);
277 }
278 }
279
280 /**
281 * Given a record containing ';'-separated fields, return an array of
282 * the fields. It is assumed that there are FIELD_COUNT fields per record.
283 */
284 static String[] getFields(String record) {
285 String[] fields = new String[FIELD_COUNT];
286 getFields(record, fields);
287 return fields;
288 }
289
290 static void dump(int plane, ObjectOutputStream out) throws Exception {
291 String version = System.getProperty("java.version");
292 System.out.println("Writing file version " + version);
293 out.writeObject(version);
294
295 long[] data = new long[0x20000];
296 long[] onechar = new long[2];
297 int j=0;
298 int begin = plane<<16;
299 int end = begin + 0xFFFF;
300 for (int i = begin; i <= end; ++i) {
301 getPackedCharacterData(i, onechar);
302 data[j++] = onechar[0];
303 data[j++] = onechar[1];
304 }
305 out.writeObject(data);
306 }
307
308 static long[] loadData(ObjectInputStream in) throws Exception {
309 String version = System.getProperty("java.version");
310 String inVersion = (String)in.readObject();
311 System.out.println("Reading file version " + inVersion);
312 System.out.println("Current version " + version);
313
314 long[] data = (long[])in.readObject();
315 if (data.length != 0x20000) {
316 System.out.println("BAD ARRAY LENGTH: " + data.length);
317 }
318 return data;
319 }
320
321 static int load(int plane, ObjectInputStream in) throws Exception {
322 long[] data = CharCheck.loadData(in);
323 CharCheck.checkData(data, plane);
324 return differences;
325 }
326
327
328 static int checkData(long[] data, int plane) {
329 long[] onechar = new long[2];
330
331 for (int i=0; i<0x10000; ++i) {
332 int c = (plane << 16) | i;
333 getPackedCharacterData(c, onechar);
334 if (data[2*i] != onechar[0] || data[2*i+1] != onechar[1]) {
335 long[] filechar = { data[2*i], data[2*i+1] };
336 showDifference(c, onechar, filechar);
337 }
338 }
339 System.out.println("Total differences: " + differences);
340 return differences;
341 }
342
343 static String hex6(long n) {
344 String q = Long.toHexString(n).toUpperCase();
345 return "000000".substring(Math.min(6, q.length())) + q;
346 }
347
348 static void showChar(int c) {
349 long[] chardata = new long[2];
350 getPackedCharacterData(c, chardata);
351 System.out.println("Current VM properties for '" + hex6(c) + "': " +
352 hex6(chardata[1]) + ' ' + hex6(chardata[0]));
353 String[] data = unpackCharacterData(chardata);
354 for (int i=0; i<data.length; ++i)
355 System.out.println(" " + escape(data[i]));
356 }
357
358 static void showFileChar(String fileName, int c) throws Exception {
359 ObjectInputStream in = new ObjectInputStream(new FileInputStream(fileName));
360 String inVersion = (String)in.readObject();
361 System.out.println("Reading file version " + inVersion);
362
363 long[] data = (long[])in.readObject();
364 if (data.length != 0x20000) {
365 System.out.println("BAD ARRAY LENGTH: " + data.length);
366 }
367 int offset = c & 0xFFFF;
368 long[] chardata = { data[2*offset], data[2*offset+1] };
369 String[] datap = unpackCharacterData(chardata);
370 System.out.println(escape("File properties for '" + hex6(c)+ "':"));
371 for (int i=0; i<datap.length; ++i)
372 System.out.println(" " + escape(datap[i]));
373 }
374
375 /**
376 * The packed character data encapsulates all the information obtainable
377 * about a character in a single numeric value.
378 *
379 * data[0]:
380 *
381 * 5 bits for getType()
382 * 6 bits for digit() -- add one
383 * 6 bits for getNumericValue() -- add two
384 * 15 bits for isXxx()
385 *
386 * 21 bits for toUpperCase()
387 *
388 *
389 * data[1]:
390 * 21 bits for toLowerCase()
391 * 21 bits for toTitleCase()
392 */
393 static void getPackedCharacterData(int c, long[] data) {
394 data[0] =
395 (long)Character.getType(c) |
396 ((long)(Character.digit(c, Character.MAX_RADIX) + 1) << 5) |
397 ((long)(Character.getNumericValue(c) + 2) << 11) |
398 (Character.isDefined(c) ? (1L<<17) : 0L) |
399 (Character.isDigit(c) ? (1L<<18) : 0L) |
400 (Character.isIdentifierIgnorable(c) ? (1L<<19) : 0L) |
401 (Character.isISOControl(c) ? (1L<<20) : 0L) |
402 (Character.isJavaIdentifierPart(c) ? (1L<<21) : 0L) |
403 (Character.isJavaIdentifierStart(c) ? (1L<<22) : 0L) |
404 (Character.isLetter(c) ? (1L<<23) : 0L) |
405 (Character.isLetterOrDigit(c) ? (1L<<24) : 0L) |
406 (Character.isLowerCase(c) ? (1L<<25) : 0L) |
407 (Character.isSpaceChar(c) ? (1L<<26) : 0L) |
408 (Character.isTitleCase(c) ? (1L<<27) : 0L) |
409 (Character.isUnicodeIdentifierPart(c) ? (1L<<28) : 0L) |
410 (Character.isUnicodeIdentifierStart(c) ? (1L<<29) : 0L) |
411 (Character.isUpperCase(c) ? (1L<<30) : 0L) |
412 (Character.isWhitespace(c) ? (1L<<31) : 0L) |
413 ((long)Character.toUpperCase(c) << 32);
414 data[1] = (long)Character.toLowerCase(c) |
415 ((long)Character.toTitleCase(c) << 21);
416 }
417
418 /**
419 * Given a long, set the bits at the given offset and length to the given value.
420 */
421 static long setBits(long data, int offset, int length, long value) {
422 long himask = -1L << (offset+length);
423 long lomask = ~(-1L << offset);
424 long lengthmask = ~(-1L << length);
425 return (data & (himask | lomask)) | ((value & lengthmask) << offset);
426 }
427
428 /**
429 * Given packed character data, change the attribute
430 * toLower
431 */
432 static void setToLower(long[] data, int value) {
433 data[0] = setBits(data[0], 48, 16, value);
434 }
435
436 /**
437 * Given packed character data, change the attribute
438 * toUpper
439 */
440 static void setToUpper(long[] data, int value) {
441 data[0] = setBits(data[0], 32, 16, value);
442 }
443
444 /**
445 * Given packed character data, change the attribute
446 * toTitle
447 */
448 static void setToTitle(long[] data, int value) {
449 data[1] = value;
450 }
451
452 /**
453 * Given packed character data, change the attribute
454 * getType
455 */
456 static void setGetType(long[] data, int value) {
457 data[0] = setBits(data[0], 0, 5, value);
458 }
459
460 /**
461 * Given packed character data, change the attribute
462 * isDefined
463 */
464 static void setIsDefined(long[] data, boolean value) {
465 data[0] = setBits(data[0], 17, 1, value?1:0);
466 }
467
468 /**
469 * Given packed character data, change the attribute
470 * isJavaIdentifierPart
471 */
472 static void setIsJavaIdentifierPart(long[] data, boolean value) {
473 data[0] = setBits(data[0], 21, 1, value?1:0);
474 }
475
476 /**
477 * Given packed character data, change the attribute
478 * isJavaIdentifierStart
479 */
480 static void setIsJavaIdentifierStart(long[] data, boolean value) {
481 data[0] = setBits(data[0], 22, 1, value?1:0);
482 }
483
484 static String[] unpackCharacterData(long[] dataL) {
485 long data = dataL[0];
486 String[] result = {
487 "type=" + getTypeName((int)(data&0x1F)),
488 "digit=" + (((data>>5)&0x3F)-1),
489 "numeric=" + (((data>>11)&0x3F)-2),
490 "isDefined=" + (((data>>17)&1)==1),
491 "isDigit=" + (((data>>18)&1)==1),
492 "isIdentifierIgnorable=" + (((data>>19)&1)==1),
493 "isISOControl=" + (((data>>20)&1)==1),
494 "isJavaIdentifierPart=" + (((data>>21)&1)==1),
495 "isJavaIdentifierStart=" + (((data>>22)&1)==1),
496 "isLetter=" + (((data>>23)&1)==1),
497 "isLetterOrDigit=" + (((data>>24)&1)==1),
498 "isLowerCase=" + (((data>>25)&1)==1),
499 "isSpaceChar=" + (((data>>26)&1)==1),
500 "isTitleCase=" + (((data>>27)&1)==1),
501 "isUnicodeIdentifierPart=" + (((data>>28)&1)==1),
502 "isUnicodeIdentifierStart=" + (((data>>29)&1)==1),
503 "isUpperCase=" + (((data>>30)&1)==1),
504 "isWhitespace=" + (((data>>31)&1)==1),
505 "toUpper=" + hex6(((int)(data>>32) & 0X1FFFFF)),
506 "toLower=" + hex6((int)(dataL[1] & 0x1FFFFF)),
507 "toTitle=" + hex6(((int)(dataL[1] >> 21) & 0x1FFFFF))
508 };
509 return result;
510 }
511
512 static String[] getCharacterData(int c) {
513 long[] data = new long[2];
514 getPackedCharacterData(c, data);
515 return unpackCharacterData(data);
516 }
517
518 static void showDifference(int c, long[] currentData, long[] fileData) {
519 System.out.println("Difference at " + hex6(c));
520 String[] current = unpackCharacterData(currentData);
521 String[] file = unpackCharacterData(fileData);
522 for (int i=0; i<current.length; ++i) {
523 if (!current[i].equals(file[i])) {
524 System.out.println(escape(" current " + current[i] +
525 ", file " + file[i]));
526 }
527 }
528 ++differences;
529 }
530
531 static String escape(String s) {
532 StringBuffer buf = new StringBuffer();
533 for (int i=0; i<s.length(); ++i) {
534 char c = s.charAt(i);
535 if (c >= 0x20 && c <= 0x7F) buf.append(c);
536 else {
537 buf.append("\\u");
538 String h = "000" + Integer.toHexString(c);
539 if (h.length() > 4) h = h.substring(h.length() - 4);
540 buf.append(h);
541 }
542 }
543 return buf.toString();
544 }
545
546 static String escape(int c) {
547 StringBuffer buf = new StringBuffer();
548 if (c >= 0x20 && c <= 0x7F) buf.append(c);
549 else {
550 buf.append("\\u");
551 String h = "000" + Integer.toHexString(c);
552 if (h.length() > 4) h = h.substring(h.length() - 4);
553 buf.append(h);
554 }
555 return buf.toString();
556 }
557 }
558
559
560 //eof