1 /* 2 * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 /* 24 * @test 25 * @bug 4052473 4052679 4055602 4066550 4067619 4068012 4068073 4070174 4070452 26 * 4070178 4070450 4070695 4070725 4070795 4071003 4071183 4071782 4072013 27 * 4072388 4072773 4075404 4084356 4087238 4092361 4094033 4094371 4098518 28 * 4099810 4103218 4103220 4103861 4112136 4113638 4113654 4117054 4122468 29 * 4122840 4139860 4156708 4175306 4215747 4209960 4290801 4900884 4942982 30 * 4518811 4945388 4936845 4794068 4461740 4965260 4984277 4826794 5032580 31 * 5102005 5074431 6182685 6208712 6277020 6245766 6351682 6386647 6379382 32 * 6414459 6455680 6498742 6558863 6488119 6547501 6497154 6558856 6481177 33 * 6379214 6485516 6486607 4225362 4494727 6533691 6531591 6531593 6570259 34 * 6509039 6609737 6610748 6645271 6507067 6873931 6450945 6645268 6646611 35 * 6645405 6650730 6910489 6573250 6870908 6585666 6716626 6914413 6916787 36 * 6919624 6998391 7019267 7020960 7025837 7020583 7036905 7066203 7101495 37 * 7003124 7085757 7028073 7171028 7189611 8000983 7195759 8004489 8006509 38 * 7114053 7074882 7040556 8008577 8013836 8021121 6192407 6931564 8027695 39 * 8017142 8037343 8055222 8042126 8074791 8075173 8080774 8129361 8134916 40 * 8145136 8145952 8164784 8037111 8081643 7037368 8178872 41 * @summary Verify locale data 42 * @modules java.base/sun.util.resources 43 * @modules jdk.localedata 44 * @run main LocaleDataTest 45 * @run main LocaleDataTest -cldr 46 * 47 */ 48 49 /* 50 * 51 * (C) Copyright Taligent, Inc. 1996, 1997 - All Rights Reserved 52 * (C) Copyright IBM Corp. 1996 - 1998 - All Rights Reserved 53 * 54 * Portions copyright (c) 2007 Sun Microsystems, Inc. 55 * All Rights Reserved. 56 * 57 * The original version of this source code and documentation 58 * is copyrighted and owned by Taligent, Inc., a wholly-owned 59 * subsidiary of IBM. These materials are provided under terms 60 * of a License Agreement between Taligent and Sun. This technology 61 * is protected by multiple US and International patents. 62 * 63 * This notice and attribution to Taligent may not be removed. 64 * Taligent is a registered trademark of Taligent, Inc. 65 * 66 * Permission to use, copy, modify, and distribute this software 67 * and its documentation for NON-COMMERCIAL purposes and without 68 * fee is hereby granted provided that this copyright notice 69 * appears in all copies. Please refer to the file "copyright.html" 70 * for further important copyright and licensing information. 71 * 72 * SUN MAKES NO REPRESENTATIONS OR WARRANTIES ABOUT THE SUITABILITY OF 73 * THE SOFTWARE, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 74 * TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 75 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT. SUN SHALL NOT BE LIABLE FOR 76 * ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING OR 77 * DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. 78 * 79 */ 80 81 /* This test is a generalized test for verifying changes to the locale data. 82 * It is driven by an external file that specifies the particular pieces of locale 83 * data to check. That file is in .properties file format: a series of key/value 84 * pairs delimited by newline characters, with the keys separated from the values 85 * by = signs. The keys are similar in syntax to a Unix pathname, with keys at 86 * successive levels of containment in the resource-data hierarchy separated by 87 * slashes. The file is in ISO 8859-1 encoding, with control characters and 88 * non-ASCII characters denoted with backslash-u escape sequences. The program also allows 89 * blank lines and comment lines to be interspersed with the data. Comment lines 90 * begin with '#'. 91 * 92 * A data file for this test would look something like this:<pre> 93 * FormatData//MonthNames/0=January 94 * FormatData//MonthNames/1=February 95 * LocaleNames//US=United States 96 * LocaleNames//FR=France 97 * FormatData/fr_FR/MonthNames/0=janvier 98 * FormatData/fr_FR/MonthNames/1=f\u00e9vrier 99 * LocaleNames/fr_FR/US=\u00c9tats-Unis 100 * LocaleNames/fr_FR/FR=France</pre> 101 * 102 * Second field which designates locale is in the form of: 103 * 1) Legacy locale notation using '_' as a locale component(language/country/variant) separator. 104 * language is a mandatory component. country and variant are optional, however, 105 * variant cannot exist without country. So for example, while "ja"/"ja_JP"/"ja_JP_JP" are valid, 106 * "_JP"/"ja__JP" are invalid. 107 * 108 * 2) BCP47 language tag notation in which we can specify language tag with '-' as a subtag 109 * separator. Language tag can be specified with '-' in locale field like this: 110 * <pre>LocaleNames/sr-Latn/SR=Surinam 111 * FormatData/sr-Latn-BA/DayNames/2=utorak</pre> 112 * 113 * The command-line syntax of this test is 114 * <tt>java LocaleDataTest [-w] [{ -s | <filename> }] [-cldr]</tt> 115 * 116 * This program always sends its results to standard output. If -w is not specified, 117 * this program prints out only the differences between the data file and the actual 118 * resource data. If -w is specified, the program prints out every entry, comment, 119 * and blank line from the data file. Where there is a difference between the data 120 * file and the resource data, the data is the data from the resources. This feature 121 * can be used to quickly generate a new data file. 122 * 123 * The user can specify an optional filename or -s. If the user specifies a filename, 124 * the program uses that file as the data file. If the user specifies -s, the program 125 * reads its input from standard input rather than from a file. If the user specifies 126 * neither, the program reads its input from a file called LocaleData in the same 127 * directory the program itself resides in. 128 * 129 * The -nothrow option prevents the program from throwing an exception when it 130 * gets an error. -w implies -nothrow. 131 * 132 * -cldr option specifies to test CLDR locale data. The default data file name for this 133 * option is "LocaleData.cldr". 134 * 135 * Other command-line options can be specified, but are ignored. 136 * 137 * It's important to note what this test will NOT test. Certain changes to the locale 138 * data are meant to have certain effects on the internationalization frameworks. For 139 * instance, we could ensure round-trip formatting/parsing integrity for the full 140 * date/time format of SimpleDateFormat by making sure that the full date and time 141 * patterns include sufficient data. The test of this is not whether changes were 142 * made to the locale data; it's whether using this data gives round-trip integrity. 143 * Likewise, changing the currency patterns to use \u00a4 instead of local currency 144 * symbols isn't something that can be tested by this test; instead, you want to 145 * actually format currency values and make sure the proper currency symbol was used. 146 * 147 * This test by itself doesn't do an exhaustive comparison of locale data. It is 148 * possible to do this manually, however: Use the GenerateKeyList tool to produce 149 * a complete list of keys for the two versions of the locales you want to compare, 150 * and then diff them. This will flag additions and deletions. Generate a data file 151 * for the base version of the data using the -w option and the output from 152 * GenerateKeyList, and then use the resultant file as the data file when you run 153 * this test against the new version of the data. 154 */ 155 156 import java.io.BufferedReader; 157 import java.io.File; 158 import java.io.FileInputStream; 159 import java.io.FilterReader; 160 import java.io.FilterWriter; 161 import java.io.IOException; 162 import java.io.InputStreamReader; 163 import java.io.OutputStreamWriter; 164 import java.io.PrintWriter; 165 import java.io.Reader; 166 import java.io.Writer; 167 import java.util.Locale; 168 import java.util.MissingResourceException; 169 import java.util.ResourceBundle; 170 import sun.util.resources.LocaleData; 171 172 public class LocaleDataTest 173 { 174 static final String TEXT_RESOURCES_PACKAGE ="sun.text.resources"; 175 static final String UTIL_RESOURCES_PACKAGE ="sun.util.resources"; 176 static final String DEFAULT_DATAFILE ="LocaleData"; 177 static String cldrSuffix = ""; 178 179 public static void main(String[] args) throws Exception { 180 181 // set up our flags and our input and output streams based on the 182 // command-line arguments (exceptions generated here will propagate out 183 // to the environment) 184 BufferedReader in = null; 185 PrintWriter out = null; 186 boolean writeNewFile = false; 187 boolean doThrow = true; 188 189 for (int i = 0; i < args.length; i++) { 190 if (args[i].equals("-w")) { 191 writeNewFile = true; 192 doThrow = false; 193 } 194 195 else if (args[i].equals("-nothrow")) 196 doThrow = false; 197 198 else if (args[i].equals("-cldr")) { 199 cldrSuffix = ".cldr"; 200 } 201 202 else if (args[i].equals("-s") && in == null) 203 in = new BufferedReader(new EscapeReader(new InputStreamReader(System.in, 204 "ISO8859_1"))); 205 else if (!args[i].startsWith("-") && in == null) 206 in = new BufferedReader(new EscapeReader(new InputStreamReader(new 207 FileInputStream(args[i]), "ISO8859_1"))); 208 } 209 if (in == null) { 210 File localeData = new File(System.getProperty("test.src", "."), DEFAULT_DATAFILE + cldrSuffix); 211 in = new BufferedReader(new EscapeReader(new InputStreamReader(new 212 FileInputStream(localeData), "ISO8859_1"))); 213 } 214 out = new PrintWriter(new EscapeWriter(new OutputStreamWriter(System.out, 215 "ISO8859_1")), true); 216 217 // perform the actual test 218 int errorCount = doTest(in, out, writeNewFile); 219 220 // write out the error count, and throw an exception out into the environment 221 // if there were any errors 222 if (errorCount != 0) { 223 if (!writeNewFile) 224 out.println("Test failed. " + errorCount + " errors."); 225 if (doThrow) 226 throw new Exception("Test failed. " + errorCount + " errors."); 227 } 228 else if (!writeNewFile) 229 out.println("Test passed."); 230 231 in.close(); 232 out.close(); 233 } 234 235 static int doTest(BufferedReader in, PrintWriter out, boolean writeNewFile) 236 throws Exception { 237 int errorCount = 0; 238 239 String key = null; 240 String expectedValue = null; 241 String line = in.readLine(); 242 while (line != null) { 243 if (line.startsWith("#") || line.length() == 0) { 244 if (writeNewFile) 245 out.println(line); 246 } 247 248 else { 249 int index = line.indexOf("="); 250 if (index == -1) { 251 key = line; 252 expectedValue = ""; 253 } 254 else { 255 key = line.substring(0, index); 256 if (index + 1 == line.length()) 257 expectedValue = ""; 258 else 259 expectedValue = line.substring(index + 1); 260 } 261 if (!processLine(key, expectedValue, out, writeNewFile)) 262 ++errorCount; 263 } 264 line = in.readLine(); 265 } 266 return errorCount; 267 } 268 269 static boolean processLine(String key, String expectedValue, PrintWriter out, 270 boolean writeNewFile) throws Exception { 271 String rbName, localeName, resTag, qualifier; 272 String language = "", country = "", variant = ""; 273 int index, oldIndex; 274 275 index = key.indexOf("/"); 276 if (index == -1 || index + 1 == key.length()) 277 throw new Exception("Malformed input file: no slashes in \"" + key + "\""); 278 rbName = key.substring(0, index); 279 280 oldIndex = index + 1; 281 index = key.indexOf("/", oldIndex); 282 if (index == -1 || index + 1 == key.length()) 283 throw new Exception("Malformed input file: \"" + key + "\" is missing locale name"); 284 localeName = key.substring(oldIndex, index); 285 boolean use_tag = localeName.indexOf("-") != -1; 286 if (use_tag == false && localeName.length() > 0) { 287 String[] locDetails = localeName.split("_"); 288 switch (locDetails.length) { 289 case 1: 290 language = locDetails[0]; 291 break; 292 case 2: 293 language = locDetails[0]; 294 country = locDetails[1]; 295 break; 296 case 3: 297 language = locDetails[0]; 298 country = locDetails[1]; 299 variant = locDetails[2]; 300 break; 301 default: 302 throw new Exception("locale not specified properly " + locDetails); 303 } 304 } 305 oldIndex = index + 1; 306 index = key.indexOf("/", oldIndex); 307 if (index == -1) 308 index = key.length(); 309 resTag = key.substring(oldIndex, index); 310 311 // TimeZone name may have "/" in it, for example "Asia/Taipei", so use "Asia\/Taipei in LocaleData. 312 if(resTag.endsWith("\\")) { 313 resTag = resTag.substring(0, resTag.length() - 1); 314 oldIndex = index; 315 index = key.indexOf("/", oldIndex + 1); 316 if (index == -1) index = key.length(); 317 resTag += key.substring(oldIndex, index); 318 } 319 320 if (index < key.length() - 1) 321 qualifier = key.substring(index + 1); 322 else 323 qualifier = ""; 324 325 String retrievedValue = null; 326 Object resource = null; 327 try { 328 String fullName = null; 329 if (rbName.equals("CalendarData") 330 || rbName.equals("CurrencyNames") 331 || rbName.equals("LocaleNames") 332 || rbName.equals("TimeZoneNames")) { 333 fullName = UTIL_RESOURCES_PACKAGE + cldrSuffix + "." + rbName; 334 } else { 335 fullName = TEXT_RESOURCES_PACKAGE + cldrSuffix + "." + rbName; 336 } 337 Locale locale; 338 if (use_tag) { 339 locale = Locale.forLanguageTag(localeName); 340 } else { 341 locale = new Locale(language, country, variant); 342 } 343 ResourceBundle bundle = LocaleData.getBundle(fullName, locale); 344 resource = bundle.getObject(resTag); 345 } 346 catch (MissingResourceException e) { 347 } 348 349 if (resource != null) { 350 if (resource instanceof String) { 351 retrievedValue = (String)resource; 352 } 353 else if (resource instanceof String[]) { 354 int element = Integer.valueOf(qualifier).intValue(); 355 String[] stringList = (String[])resource; 356 if (element >= 0 && element < stringList.length) 357 retrievedValue = stringList[element]; 358 } 359 else if (resource instanceof String[][]) { 360 String[][] stringArray = (String[][])resource; 361 int slash = qualifier.indexOf("/"); 362 if (slash == -1) { 363 for (int i = 0; i < stringArray.length; i++) { 364 if (stringArray[i][0].equals(qualifier)) 365 retrievedValue = stringArray[i][1]; 366 } 367 } 368 else { 369 int row = Integer.valueOf(qualifier.substring(0, slash)).intValue(); 370 int column = Integer.valueOf(qualifier.substring(slash + 1)).intValue(); 371 if (row >= 0 && row < stringArray.length && column >= 0 && column < 372 stringArray[row].length) 373 retrievedValue = stringArray[row][column]; 374 } 375 } 376 } 377 378 if (retrievedValue == null || !retrievedValue.equals(expectedValue)) { 379 if (retrievedValue == null) 380 retrievedValue = "<MISSING!>"; 381 382 if (writeNewFile) 383 out.println(key + "=" + retrievedValue); 384 else { 385 out.println("Mismatch in " + key + ":"); 386 out.println(" file = \"" + expectedValue + "\""); 387 out.println(" jvm = \"" + retrievedValue + "\""); 388 } 389 return false; 390 } 391 else { 392 if (writeNewFile) 393 out.println(key + "=" + expectedValue); 394 } 395 return true; 396 } 397 } 398 399 class EscapeReader extends FilterReader { 400 public EscapeReader(Reader in) { 401 super(in); 402 } 403 404 public int read() throws IOException { 405 if (buffer != null) { 406 String b = buffer.toString(); 407 int result = b.charAt(0); 408 if (b.length() > 1) 409 buffer = new StringBuffer(b.substring(1)); 410 else 411 buffer = null; 412 return result; 413 } 414 else { 415 int result = super.read(); 416 if (result != '\\') 417 return result; 418 else { 419 buffer = new StringBuffer(); 420 result = super.read(); 421 buffer.append((char)result); 422 if (result == 'u') { 423 for (int i = 0; i < 4; i++) { 424 result = super.read(); 425 if (result == -1) 426 break; 427 buffer.append((char)result); 428 } 429 String number = buffer.toString().substring(1); 430 result = Integer.parseInt(number, 16); 431 buffer = null; 432 return result; 433 } 434 return '\\'; 435 } 436 } 437 } 438 439 public int read(char[] cbuf, int start, int len) throws IOException { 440 int p = start; 441 int end = start + len; 442 int c = 0; 443 while (c != -1 && p < end) { 444 c = read(); 445 if (c != -1) 446 cbuf[p++] = (char)c; 447 } 448 if (c == -1 && p == start) 449 return -1; 450 else 451 return p - start; 452 } 453 454 private StringBuffer buffer = null; 455 } 456 457 class EscapeWriter extends FilterWriter { 458 public EscapeWriter(Writer out) { 459 super(out); 460 } 461 462 public void write(int c) throws IOException { 463 if ((c >= ' ' && c <= '\u007e') || c == '\r' || c == '\n') 464 super.write(c); 465 else { 466 super.write('\\'); 467 super.write('u'); 468 String number = Integer.toHexString(c); 469 if (number.length() < 4) 470 number = zeros.substring(0, 4 - number.length()) + number; 471 super.write(number.charAt(0)); 472 super.write(number.charAt(1)); 473 super.write(number.charAt(2)); 474 super.write(number.charAt(3)); 475 } 476 } 477 478 public void write(char[] cbuf, int off, int len) throws IOException { 479 int end = off + len; 480 while (off < end) 481 write(cbuf[off++]); 482 } 483 484 public void write(String str, int off, int len) throws IOException { 485 int end = off + len; 486 while (off < end) 487 write(str.charAt(off++)); 488 } 489 490 private static String zeros = "0000"; 491 }