1 /* 2 * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "utilities/utf8.hpp" 27 28 // Assume the utf8 string is in legal form and has been 29 // checked in the class file parser/format checker. 30 template<typename T> char* UTF8::next(const char* str, T* value) { 31 unsigned const char *ptr = (const unsigned char *)str; 32 unsigned char ch, ch2, ch3; 33 int length = -1; /* bad length */ 34 jchar result; 35 switch ((ch = ptr[0]) >> 4) { 36 default: 37 result = ch; 38 length = 1; 39 break; 40 41 case 0x8: case 0x9: case 0xA: case 0xB: case 0xF: 42 /* Shouldn't happen. */ 43 break; 44 45 case 0xC: case 0xD: 46 /* 110xxxxx 10xxxxxx */ 47 if (((ch2 = ptr[1]) & 0xC0) == 0x80) { 48 unsigned char high_five = ch & 0x1F; 49 unsigned char low_six = ch2 & 0x3F; 50 result = (high_five << 6) + low_six; 51 length = 2; 52 break; 53 } 54 break; 55 56 case 0xE: 57 /* 1110xxxx 10xxxxxx 10xxxxxx */ 58 if (((ch2 = ptr[1]) & 0xC0) == 0x80) { 59 if (((ch3 = ptr[2]) & 0xC0) == 0x80) { 60 unsigned char high_four = ch & 0x0f; 61 unsigned char mid_six = ch2 & 0x3f; 62 unsigned char low_six = ch3 & 0x3f; 63 result = (((high_four << 6) + mid_six) << 6) + low_six; 64 length = 3; 65 } 66 } 67 break; 68 } /* end of switch */ 69 70 if (length <= 0) { 71 *value = (T)ptr[0]; /* default bad result; */ 72 return (char*)(ptr + 1); // make progress somehow 73 } 74 75 *value = (T)result; 76 77 // The assert is correct but the .class file is wrong 78 // assert(UNICODE::utf8_size(result) == length, "checking reverse computation"); 79 return (char *)(ptr + length); 80 } 81 82 char* UTF8::next_character(const char* str, jint* value) { 83 unsigned const char *ptr = (const unsigned char *)str; 84 /* See if it's legal supplementary character: 85 11101101 1010xxxx 10xxxxxx 11101101 1011xxxx 10xxxxxx */ 86 if (is_supplementary_character(ptr)) { 87 *value = get_supplementary_character(ptr); 88 return (char *)(ptr + 6); 89 } 90 jchar result; 91 char* next_ch = next(str, &result); 92 *value = result; 93 return next_ch; 94 } 95 96 // Count bytes of the form 10xxxxxx and deduct this count 97 // from the total byte count. The utf8 string must be in 98 // legal form which has been verified in the format checker. 99 int UTF8::unicode_length(const char* str, int len, bool& is_latin1, bool& has_multibyte) { 100 int num_chars = len; 101 has_multibyte = false; 102 is_latin1 = true; 103 unsigned char prev = 0; 104 for (int i = 0; i < len; i++) { 105 unsigned char c = str[i]; 106 if ((c & 0xC0) == 0x80) { 107 // Multibyte, check if valid latin1 character. 108 has_multibyte = true; 109 if (prev > 0xC3) { 110 is_latin1 = false; 111 } 112 --num_chars; 113 } 114 prev = c; 115 } 116 return num_chars; 117 } 118 119 // Count bytes of the utf8 string except those in form 120 // 10xxxxxx which only appear in multibyte characters. 121 // The utf8 string must be in legal form and has been 122 // verified in the format checker. 123 int UTF8::unicode_length(const char* str, bool& is_latin1, bool& has_multibyte) { 124 int num_chars = 0; 125 has_multibyte = false; 126 is_latin1 = true; 127 unsigned char prev = 0; 128 for (const char* p = str; *p; p++) { 129 unsigned char c = (*p); 130 if ((c & 0xC0) == 0x80) { 131 // Multibyte, check if valid latin1 character. 132 has_multibyte = true; 133 if (prev > 0xC3) { 134 is_latin1 = false; 135 } 136 } else { 137 num_chars++; 138 } 139 prev = c; 140 } 141 return num_chars; 142 } 143 144 // Writes a jchar as utf8 and returns the end 145 static u_char* utf8_write(u_char* base, jchar ch) { 146 if ((ch != 0) && (ch <=0x7f)) { 147 base[0] = (u_char) ch; 148 return base + 1; 149 } 150 151 if (ch <= 0x7FF) { 152 /* 11 bits or less. */ 153 unsigned char high_five = ch >> 6; 154 unsigned char low_six = ch & 0x3F; 155 base[0] = high_five | 0xC0; /* 110xxxxx */ 156 base[1] = low_six | 0x80; /* 10xxxxxx */ 157 return base + 2; 158 } 159 /* possibly full 16 bits. */ 160 char high_four = ch >> 12; 161 char mid_six = (ch >> 6) & 0x3F; 162 char low_six = ch & 0x3f; 163 base[0] = high_four | 0xE0; /* 1110xxxx */ 164 base[1] = mid_six | 0x80; /* 10xxxxxx */ 165 base[2] = low_six | 0x80; /* 10xxxxxx */ 166 return base + 3; 167 } 168 169 template<typename T> void UTF8::convert_to_unicode(const char* utf8_str, T* unicode_str, int unicode_length) { 170 unsigned char ch; 171 const char *ptr = utf8_str; 172 int index = 0; 173 174 /* ASCII case loop optimization */ 175 for (; index < unicode_length; index++) { 176 if((ch = ptr[0]) > 0x7F) { break; } 177 unicode_str[index] = (T)ch; 178 ptr = (const char *)(ptr + 1); 179 } 180 181 for (; index < unicode_length; index++) { 182 ptr = UTF8::next(ptr, &unicode_str[index]); 183 } 184 } 185 186 // Explicit instantiation for all supported string types. 187 template char* UTF8::next<jchar>(const char* str, jchar* value); 188 template char* UTF8::next<jbyte>(const char* str, jbyte* value); 189 template void UTF8::convert_to_unicode<jchar>(const char* utf8_str, jchar* unicode_str, int unicode_length); 190 template void UTF8::convert_to_unicode<jbyte>(const char* utf8_str, jbyte* unicode_str, int unicode_length); 191 192 // returns the quoted ascii length of a 0-terminated utf8 string 193 int UTF8::quoted_ascii_length(const char* utf8_str, int utf8_length) { 194 const char *ptr = utf8_str; 195 const char* end = ptr + utf8_length; 196 int result = 0; 197 while (ptr < end) { 198 jchar c; 199 ptr = UTF8::next(ptr, &c); 200 if (c >= 32 && c < 127) { 201 result++; 202 } else { 203 result += 6; 204 } 205 } 206 return result; 207 } 208 209 // converts a utf8 string to quoted ascii 210 void UTF8::as_quoted_ascii(const char* utf8_str, int utf8_length, char* buf, int buflen) { 211 const char *ptr = utf8_str; 212 const char *utf8_end = ptr + utf8_length; 213 char* p = buf; 214 char* end = buf + buflen; 215 while (ptr < utf8_end) { 216 jchar c; 217 ptr = UTF8::next(ptr, &c); 218 if (c >= 32 && c < 127) { 219 if (p + 1 >= end) break; // string is truncated 220 *p++ = (char)c; 221 } else { 222 if (p + 6 >= end) break; // string is truncated 223 sprintf(p, "\\u%04x", c); 224 p += 6; 225 } 226 } 227 assert(p < end, "sanity"); 228 *p = '\0'; 229 } 230 231 232 const char* UTF8::from_quoted_ascii(const char* quoted_ascii_str) { 233 const char *ptr = quoted_ascii_str; 234 char* result = NULL; 235 while (*ptr != '\0') { 236 char c = *ptr; 237 if (c < 32 || c >= 127) break; 238 } 239 if (*ptr == '\0') { 240 // nothing to do so return original string 241 return quoted_ascii_str; 242 } 243 // everything up to this point was ok. 244 int length = ptr - quoted_ascii_str; 245 char* buffer = NULL; 246 for (int round = 0; round < 2; round++) { 247 while (*ptr != '\0') { 248 if (*ptr != '\\') { 249 if (buffer != NULL) { 250 buffer[length] = *ptr; 251 } 252 length++; 253 } else { 254 switch (ptr[1]) { 255 case 'u': { 256 ptr += 2; 257 jchar value=0; 258 for (int i=0; i<4; i++) { 259 char c = *ptr++; 260 switch (c) { 261 case '0': case '1': case '2': case '3': case '4': 262 case '5': case '6': case '7': case '8': case '9': 263 value = (value << 4) + c - '0'; 264 break; 265 case 'a': case 'b': case 'c': 266 case 'd': case 'e': case 'f': 267 value = (value << 4) + 10 + c - 'a'; 268 break; 269 case 'A': case 'B': case 'C': 270 case 'D': case 'E': case 'F': 271 value = (value << 4) + 10 + c - 'A'; 272 break; 273 default: 274 ShouldNotReachHere(); 275 } 276 } 277 if (buffer == NULL) { 278 char utf8_buffer[4]; 279 char* next = (char*)utf8_write((u_char*)utf8_buffer, value); 280 length += next - utf8_buffer; 281 } else { 282 char* next = (char*)utf8_write((u_char*)&buffer[length], value); 283 length += next - &buffer[length]; 284 } 285 break; 286 } 287 case 't': if (buffer != NULL) buffer[length] = '\t'; ptr += 2; length++; break; 288 case 'n': if (buffer != NULL) buffer[length] = '\n'; ptr += 2; length++; break; 289 case 'r': if (buffer != NULL) buffer[length] = '\r'; ptr += 2; length++; break; 290 case 'f': if (buffer != NULL) buffer[length] = '\f'; ptr += 2; length++; break; 291 default: 292 ShouldNotReachHere(); 293 } 294 } 295 } 296 if (round == 0) { 297 buffer = NEW_RESOURCE_ARRAY(char, length + 1); 298 ptr = quoted_ascii_str; 299 } else { 300 buffer[length] = '\0'; 301 } 302 } 303 return buffer; 304 } 305 306 307 // Returns NULL if 'c' it not found. This only works as long 308 // as 'c' is an ASCII character 309 const jbyte* UTF8::strrchr(const jbyte* base, int length, jbyte c) { 310 assert(length >= 0, "sanity check"); 311 assert(c >= 0, "does not work for non-ASCII characters"); 312 // Skip backwards in string until 'c' is found or end is reached 313 while(--length >= 0 && base[length] != c); 314 return (length < 0) ? NULL : &base[length]; 315 } 316 317 bool UTF8::equal(const jbyte* base1, int length1, const jbyte* base2, int length2) { 318 // Length must be the same 319 if (length1 != length2) return false; 320 for (int i = 0; i < length1; i++) { 321 if (base1[i] != base2[i]) return false; 322 } 323 return true; 324 } 325 326 bool UTF8::is_supplementary_character(const unsigned char* str) { 327 return ((str[0] & 0xFF) == 0xED) && ((str[1] & 0xF0) == 0xA0) && ((str[2] & 0xC0) == 0x80) 328 && ((str[3] & 0xFF) == 0xED) && ((str[4] & 0xF0) == 0xB0) && ((str[5] & 0xC0) == 0x80); 329 } 330 331 jint UTF8::get_supplementary_character(const unsigned char* str) { 332 return 0x10000 + ((str[1] & 0x0f) << 16) + ((str[2] & 0x3f) << 10) 333 + ((str[4] & 0x0f) << 6) + (str[5] & 0x3f); 334 } 335 336 bool UTF8::is_legal_utf8(const unsigned char* buffer, int length, 337 bool version_leq_47) { 338 int i = 0; 339 int count = length >> 2; 340 for (int k=0; k<count; k++) { 341 unsigned char b0 = buffer[i]; 342 unsigned char b1 = buffer[i+1]; 343 unsigned char b2 = buffer[i+2]; 344 unsigned char b3 = buffer[i+3]; 345 // For an unsigned char v, 346 // (v | v - 1) is < 128 (highest bit 0) for 0 < v < 128; 347 // (v | v - 1) is >= 128 (highest bit 1) for v == 0 or v >= 128. 348 unsigned char res = b0 | b0 - 1 | 349 b1 | b1 - 1 | 350 b2 | b2 - 1 | 351 b3 | b3 - 1; 352 if (res >= 128) break; 353 i += 4; 354 } 355 for(; i < length; i++) { 356 unsigned short c; 357 // no embedded zeros 358 if (buffer[i] == 0) return false; 359 if(buffer[i] < 128) { 360 continue; 361 } 362 if ((i + 5) < length) { // see if it's legal supplementary character 363 if (UTF8::is_supplementary_character(&buffer[i])) { 364 c = UTF8::get_supplementary_character(&buffer[i]); 365 i += 5; 366 continue; 367 } 368 } 369 switch (buffer[i] >> 4) { 370 default: break; 371 case 0x8: case 0x9: case 0xA: case 0xB: case 0xF: 372 return false; 373 case 0xC: case 0xD: // 110xxxxx 10xxxxxx 374 c = (buffer[i] & 0x1F) << 6; 375 i++; 376 if ((i < length) && ((buffer[i] & 0xC0) == 0x80)) { 377 c += buffer[i] & 0x3F; 378 if (version_leq_47 || c == 0 || c >= 0x80) { 379 break; 380 } 381 } 382 return false; 383 case 0xE: // 1110xxxx 10xxxxxx 10xxxxxx 384 c = (buffer[i] & 0xF) << 12; 385 i += 2; 386 if ((i < length) && ((buffer[i-1] & 0xC0) == 0x80) && ((buffer[i] & 0xC0) == 0x80)) { 387 c += ((buffer[i-1] & 0x3F) << 6) + (buffer[i] & 0x3F); 388 if (version_leq_47 || c >= 0x800) { 389 break; 390 } 391 } 392 return false; 393 } // end of switch 394 } // end of for 395 return true; 396 } 397 398 //------------------------------------------------------------------------------------- 399 400 bool UNICODE::is_latin1(jchar c) { 401 return (c <= 0x00FF); 402 } 403 404 bool UNICODE::is_latin1(jchar* base, int length) { 405 for (int index = 0; index < length; index++) { 406 if (base[index] > 0x00FF) { 407 return false; 408 } 409 } 410 return true; 411 } 412 413 int UNICODE::utf8_size(jchar c) { 414 if ((0x0001 <= c) && (c <= 0x007F)) return 1; 415 if (c <= 0x07FF) return 2; 416 return 3; 417 } 418 419 int UNICODE::utf8_size(jbyte c) { 420 if (c >= 0x0001) return 1; 421 return 2; 422 } 423 424 int UNICODE::utf8_length(jchar* base, int length) { 425 int result = 0; 426 for (int index = 0; index < length; index++) { 427 jchar c = base[index]; 428 if ((0x0001 <= c) && (c <= 0x007F)) result += 1; 429 else if (c <= 0x07FF) result += 2; 430 else result += 3; 431 } 432 return result; 433 } 434 435 int UNICODE::utf8_length(jbyte* base, int length) { 436 int result = 0; 437 for (int index = 0; index < length; index++) { 438 jbyte c = base[index]; 439 result += utf8_size(c); 440 } 441 return result; 442 } 443 444 char* UNICODE::as_utf8(jchar* base, int length) { 445 int utf8_len = utf8_length(base, length); 446 u_char* buf = NEW_RESOURCE_ARRAY(u_char, utf8_len + 1); 447 char* result = as_utf8(base, length, (char*) buf, utf8_len + 1); 448 assert((int) strlen(result) == utf8_len, "length prediction must be correct"); 449 return result; 450 } 451 452 char* UNICODE::as_utf8(jbyte* base, int length) { 453 int utf8_len = utf8_length(base, length); 454 u_char* result = NEW_RESOURCE_ARRAY(u_char, utf8_len + 1); 455 u_char* p = result; 456 if (utf8_len == length) { 457 for (int index = 0; index < length; index++) { 458 *p++ = base[index]; 459 } 460 } else { 461 // Unicode string contains U+0000 which should 462 // be encoded as 0xC080 in "modified" UTF8. 463 for (int index = 0; index < length; index++) { 464 p = utf8_write(p, ((jchar) base[index]) & 0xff); 465 } 466 } 467 *p = '\0'; 468 assert(p == &result[utf8_len], "length prediction must be correct"); 469 return (char*) result; 470 } 471 472 char* UNICODE::as_utf8(jchar* base, int length, char* buf, int buflen) { 473 u_char* p = (u_char*)buf; 474 for (int index = 0; index < length; index++) { 475 jchar c = base[index]; 476 buflen -= utf8_size(c); 477 if (buflen <= 0) break; // string is truncated 478 p = utf8_write(p, c); 479 } 480 *p = '\0'; 481 return buf; 482 } 483 484 char* UNICODE::as_utf8(jbyte* base, int length, char* buf, int buflen) { 485 u_char* p = (u_char*)buf; 486 u_char* end = (u_char*)buf + buflen; 487 for (int index = 0; index < length; index++) { 488 jbyte c = base[index]; 489 int sz = utf8_size(c); 490 buflen -= sz; 491 if (buflen <= 0) break; // string is truncated 492 if (sz == 1) { 493 *p++ = c; 494 } else { 495 // Unicode string contains U+0000 which should 496 // be encoded as 0xC080 in "modified" UTF8. 497 p = utf8_write(p, ((jchar) c) & 0xff); 498 } 499 } 500 *p = '\0'; 501 return buf; 502 } 503 504 void UNICODE::convert_to_utf8(const jchar* base, int length, char* utf8_buffer) { 505 for(int index = 0; index < length; index++) { 506 utf8_buffer = (char*)utf8_write((u_char*)utf8_buffer, base[index]); 507 } 508 *utf8_buffer = '\0'; 509 } 510 511 // returns the quoted ascii length of a unicode string 512 template<typename T> 513 int UNICODE::quoted_ascii_length(T* base, int length) { 514 int result = 0; 515 for (int i = 0; i < length; i++) { 516 T c = base[i]; 517 if (c >= 32 && c < 127) { 518 result++; 519 } else { 520 result += 6; 521 } 522 } 523 return result; 524 } 525 526 // converts a unicode string to quoted ascii 527 template<typename T> 528 void UNICODE::as_quoted_ascii(const T* base, int length, char* buf, int buflen) { 529 char* p = buf; 530 char* end = buf + buflen; 531 for (int index = 0; index < length; index++) { 532 T c = base[index]; 533 if (c >= 32 && c < 127) { 534 if (p + 1 >= end) break; // string is truncated 535 *p++ = (char)c; 536 } else { 537 if (p + 6 >= end) break; // string is truncated 538 sprintf(p, "\\u%04x", c); 539 p += 6; 540 } 541 } 542 *p = '\0'; 543 } 544 545 // Explicit instantiation for all supported types. 546 template int UNICODE::quoted_ascii_length<jbyte>(jbyte* base, int length); 547 template int UNICODE::quoted_ascii_length<jchar>(jchar* base, int length); 548 template void UNICODE::as_quoted_ascii<jbyte>(const jbyte* base, int length, char* buf, int buflen); 549 template void UNICODE::as_quoted_ascii<jchar>(const jchar* base, int length, char* buf, int buflen);