1 /* 2 * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/altHashing.hpp" 27 #include "classfile/javaClasses.inline.hpp" 28 #include "gc/g1/g1CollectedHeap.inline.hpp" 29 #include "gc/g1/g1SATBCardTableModRefBS.hpp" 30 #include "gc/g1/g1StringDedup.hpp" 31 #include "gc/g1/g1StringDedupTable.hpp" 32 #include "gc/shared/gcLocker.hpp" 33 #include "logging/log.hpp" 34 #include "memory/padded.inline.hpp" 35 #include "oops/oop.inline.hpp" 36 #include "oops/typeArrayOop.hpp" 37 #include "runtime/mutexLocker.hpp" 38 39 // 40 // Freelist in the deduplication table entry cache. Links table 41 // entries together using their _next fields. 42 // 43 class G1StringDedupEntryFreeList : public CHeapObj<mtGC> { 44 private: 45 G1StringDedupEntry* _list; 46 size_t _length; 47 48 public: 49 G1StringDedupEntryFreeList() : 50 _list(NULL), 51 _length(0) { 52 } 53 54 void add(G1StringDedupEntry* entry) { 55 entry->set_next(_list); 56 _list = entry; 57 _length++; 58 } 59 60 G1StringDedupEntry* remove() { 61 G1StringDedupEntry* entry = _list; 62 if (entry != NULL) { 63 _list = entry->next(); 64 _length--; 65 } 66 return entry; 67 } 68 69 size_t length() { 70 return _length; 71 } 72 }; 73 74 // 75 // Cache of deduplication table entries. This cache provides fast allocation and 76 // reuse of table entries to lower the pressure on the underlying allocator. 77 // But more importantly, it provides fast/deferred freeing of table entries. This 78 // is important because freeing of table entries is done during stop-the-world 79 // phases and it is not uncommon for large number of entries to be freed at once. 80 // Tables entries that are freed during these phases are placed onto a freelist in 81 // the cache. The deduplication thread, which executes in a concurrent phase, will 82 // later reuse or free the underlying memory for these entries. 83 // 84 // The cache allows for single-threaded allocations and multi-threaded frees. 85 // Allocations are synchronized by StringDedupTable_lock as part of a table 86 // modification. 87 // 88 class G1StringDedupEntryCache : public CHeapObj<mtGC> { 89 private: 90 // One freelist per GC worker to allow lock less freeing of 91 // entries while doing a parallel scan of the table. Using 92 // PaddedEnd to avoid false sharing. 93 PaddedEnd<G1StringDedupEntryFreeList>* _lists; 94 size_t _nlists; 95 96 public: 97 G1StringDedupEntryCache(); 98 ~G1StringDedupEntryCache(); 99 100 // Get a table entry from the cache freelist, or allocate a new 101 // entry if the cache is empty. 102 G1StringDedupEntry* alloc(); 103 104 // Insert a table entry into the cache freelist. 105 void free(G1StringDedupEntry* entry, uint worker_id); 106 107 // Returns current number of entries in the cache. 108 size_t size(); 109 110 // If the cache has grown above the given max size, trim it down 111 // and deallocate the memory occupied by trimmed of entries. 112 void trim(size_t max_size); 113 }; 114 115 G1StringDedupEntryCache::G1StringDedupEntryCache() { 116 _nlists = ParallelGCThreads; 117 _lists = PaddedArray<G1StringDedupEntryFreeList, mtGC>::create_unfreeable((uint)_nlists); 118 } 119 120 G1StringDedupEntryCache::~G1StringDedupEntryCache() { 121 ShouldNotReachHere(); 122 } 123 124 G1StringDedupEntry* G1StringDedupEntryCache::alloc() { 125 for (size_t i = 0; i < _nlists; i++) { 126 G1StringDedupEntry* entry = _lists[i].remove(); 127 if (entry != NULL) { 128 return entry; 129 } 130 } 131 return new G1StringDedupEntry(); 132 } 133 134 void G1StringDedupEntryCache::free(G1StringDedupEntry* entry, uint worker_id) { 135 assert(entry->obj() != NULL, "Double free"); 136 assert(worker_id < _nlists, "Invalid worker id"); 137 entry->set_obj(NULL); 138 entry->set_hash(0); 139 _lists[worker_id].add(entry); 140 } 141 142 size_t G1StringDedupEntryCache::size() { 143 size_t size = 0; 144 for (size_t i = 0; i < _nlists; i++) { 145 size += _lists[i].length(); 146 } 147 return size; 148 } 149 150 void G1StringDedupEntryCache::trim(size_t max_size) { 151 size_t cache_size = 0; 152 for (size_t i = 0; i < _nlists; i++) { 153 G1StringDedupEntryFreeList* list = &_lists[i]; 154 cache_size += list->length(); 155 while (cache_size > max_size) { 156 G1StringDedupEntry* entry = list->remove(); 157 assert(entry != NULL, "Should not be null"); 158 cache_size--; 159 delete entry; 160 } 161 } 162 } 163 164 G1StringDedupTable* G1StringDedupTable::_table = NULL; 165 G1StringDedupEntryCache* G1StringDedupTable::_entry_cache = NULL; 166 167 const size_t G1StringDedupTable::_min_size = (1 << 10); // 1024 168 const size_t G1StringDedupTable::_max_size = (1 << 24); // 16777216 169 const double G1StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load 170 const double G1StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load 171 const double G1StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size 172 const uintx G1StringDedupTable::_rehash_multiple = 60; // Hash bucket has 60 times more collisions than expected 173 const uintx G1StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor); 174 175 uintx G1StringDedupTable::_entries_added = 0; 176 uintx G1StringDedupTable::_entries_removed = 0; 177 uintx G1StringDedupTable::_resize_count = 0; 178 uintx G1StringDedupTable::_rehash_count = 0; 179 180 G1StringDedupTable::G1StringDedupTable(size_t size, jint hash_seed) : 181 _size(size), 182 _entries(0), 183 _grow_threshold((uintx)(size * _grow_load_factor)), 184 _shrink_threshold((uintx)(size * _shrink_load_factor)), 185 _rehash_needed(false), 186 _hash_seed(hash_seed) { 187 assert(is_power_of_2(size), "Table size must be a power of 2"); 188 _buckets = NEW_C_HEAP_ARRAY(G1StringDedupEntry*, _size, mtGC); 189 memset(_buckets, 0, _size * sizeof(G1StringDedupEntry*)); 190 } 191 192 G1StringDedupTable::~G1StringDedupTable() { 193 FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets); 194 } 195 196 void G1StringDedupTable::create() { 197 assert(_table == NULL, "One string deduplication table allowed"); 198 _entry_cache = new G1StringDedupEntryCache(); 199 _table = new G1StringDedupTable(_min_size); 200 } 201 202 void G1StringDedupTable::add(typeArrayOop value, bool latin1, unsigned int hash, G1StringDedupEntry** list) { 203 G1StringDedupEntry* entry = _entry_cache->alloc(); 204 entry->set_obj(value); 205 entry->set_hash(hash); 206 entry->set_latin1(latin1); 207 entry->set_next(*list); 208 *list = entry; 209 _entries++; 210 } 211 212 void G1StringDedupTable::remove(G1StringDedupEntry** pentry, uint worker_id) { 213 G1StringDedupEntry* entry = *pentry; 214 *pentry = entry->next(); 215 _entry_cache->free(entry, worker_id); 216 } 217 218 void G1StringDedupTable::transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest) { 219 G1StringDedupEntry* entry = *pentry; 220 *pentry = entry->next(); 221 unsigned int hash = entry->hash(); 222 size_t index = dest->hash_to_index(hash); 223 G1StringDedupEntry** list = dest->bucket(index); 224 entry->set_next(*list); 225 *list = entry; 226 } 227 228 bool G1StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) { 229 return (value1 == value2 || 230 (value1->length() == value2->length() && 231 (!memcmp(value1->base(T_BYTE), 232 value2->base(T_BYTE), 233 value1->length() * sizeof(jbyte))))); 234 } 235 236 typeArrayOop G1StringDedupTable::lookup(typeArrayOop value, bool latin1, unsigned int hash, 237 G1StringDedupEntry** list, uintx &count) { 238 for (G1StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) { 239 if (entry->hash() == hash && entry->latin1() == latin1) { 240 typeArrayOop existing_value = entry->obj(); 241 if (equals(value, existing_value)) { 242 // Match found 243 return existing_value; 244 } 245 } 246 count++; 247 } 248 249 // Not found 250 return NULL; 251 } 252 253 typeArrayOop G1StringDedupTable::lookup_or_add_inner(typeArrayOop value, bool latin1, unsigned int hash) { 254 size_t index = hash_to_index(hash); 255 G1StringDedupEntry** list = bucket(index); 256 uintx count = 0; 257 258 // Lookup in list 259 typeArrayOop existing_value = lookup(value, latin1, hash, list, count); 260 261 // Check if rehash is needed 262 if (count > _rehash_threshold) { 263 _rehash_needed = true; 264 } 265 266 if (existing_value == NULL) { 267 // Not found, add new entry 268 add(value, latin1, hash, list); 269 270 // Update statistics 271 _entries_added++; 272 } 273 274 return existing_value; 275 } 276 277 unsigned int G1StringDedupTable::hash_code(typeArrayOop value, bool latin1) { 278 unsigned int hash; 279 int length = value->length(); 280 if (latin1) { 281 const jbyte* data = (jbyte*)value->base(T_BYTE); 282 if (use_java_hash()) { 283 hash = java_lang_String::hash_code(data, length); 284 } else { 285 hash = AltHashing::murmur3_32(_table->_hash_seed, data, length); 286 } 287 } else { 288 length /= sizeof(jchar) / sizeof(jbyte); // Convert number of bytes to number of chars 289 const jchar* data = (jchar*)value->base(T_CHAR); 290 if (use_java_hash()) { 291 hash = java_lang_String::hash_code(data, length); 292 } else { 293 hash = AltHashing::murmur3_32(_table->_hash_seed, data, length); 294 } 295 } 296 297 return hash; 298 } 299 300 void G1StringDedupTable::deduplicate(oop java_string, G1StringDedupStat& stat) { 301 assert(java_lang_String::is_instance(java_string), "Must be a string"); 302 NoSafepointVerifier nsv; 303 304 stat.inc_inspected(); 305 306 typeArrayOop value = java_lang_String::value(java_string); 307 if (value == NULL) { 308 // String has no value 309 stat.inc_skipped(); 310 return; 311 } 312 313 bool latin1 = java_lang_String::is_latin1(java_string); 314 unsigned int hash = 0; 315 316 if (use_java_hash()) { 317 // Get hash code from cache 318 hash = java_lang_String::hash(java_string); 319 } 320 321 if (hash == 0) { 322 // Compute hash 323 hash = hash_code(value, latin1); 324 stat.inc_hashed(); 325 326 if (use_java_hash() && hash != 0) { 327 // Store hash code in cache 328 java_lang_String::set_hash(java_string, hash); 329 } 330 } 331 332 typeArrayOop existing_value = lookup_or_add(value, latin1, hash); 333 if (existing_value == value) { 334 // Same value, already known 335 stat.inc_known(); 336 return; 337 } 338 339 // Get size of value array 340 uintx size_in_bytes = value->size() * HeapWordSize; 341 stat.inc_new(size_in_bytes); 342 343 if (existing_value != NULL) { 344 // Enqueue the reference to make sure it is kept alive. Concurrent mark might 345 // otherwise declare it dead if there are no other strong references to this object. 346 G1SATBCardTableModRefBS::enqueue(existing_value); 347 348 // Existing value found, deduplicate string 349 java_lang_String::set_value(java_string, existing_value); 350 351 if (G1CollectedHeap::heap()->is_in_young(value)) { 352 stat.inc_deduped_young(size_in_bytes); 353 } else { 354 stat.inc_deduped_old(size_in_bytes); 355 } 356 } 357 } 358 359 G1StringDedupTable* G1StringDedupTable::prepare_resize() { 360 size_t size = _table->_size; 361 362 // Check if the hashtable needs to be resized 363 if (_table->_entries > _table->_grow_threshold) { 364 // Grow table, double the size 365 size *= 2; 366 if (size > _max_size) { 367 // Too big, don't resize 368 return NULL; 369 } 370 } else if (_table->_entries < _table->_shrink_threshold) { 371 // Shrink table, half the size 372 size /= 2; 373 if (size < _min_size) { 374 // Too small, don't resize 375 return NULL; 376 } 377 } else if (StringDeduplicationResizeALot) { 378 // Force grow 379 size *= 2; 380 if (size > _max_size) { 381 // Too big, force shrink instead 382 size /= 4; 383 } 384 } else { 385 // Resize not needed 386 return NULL; 387 } 388 389 // Update statistics 390 _resize_count++; 391 392 // Allocate the new table. The new table will be populated by workers 393 // calling unlink_or_oops_do() and finally installed by finish_resize(). 394 return new G1StringDedupTable(size, _table->_hash_seed); 395 } 396 397 void G1StringDedupTable::finish_resize(G1StringDedupTable* resized_table) { 398 assert(resized_table != NULL, "Invalid table"); 399 400 resized_table->_entries = _table->_entries; 401 402 // Free old table 403 delete _table; 404 405 // Install new table 406 _table = resized_table; 407 } 408 409 void G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) { 410 // The table is divided into partitions to allow lock-less parallel processing by 411 // multiple worker threads. A worker thread first claims a partition, which ensures 412 // exclusive access to that part of the table, then continues to process it. To allow 413 // shrinking of the table in parallel we also need to make sure that the same worker 414 // thread processes all partitions where entries will hash to the same destination 415 // partition. Since the table size is always a power of two and we always shrink by 416 // dividing the table in half, we know that for a given partition there is only one 417 // other partition whoes entries will hash to the same destination partition. That 418 // other partition is always the sibling partition in the second half of the table. 419 // For example, if the table is divided into 8 partitions, the sibling of partition 0 420 // is partition 4, the sibling of partition 1 is partition 5, etc. 421 size_t table_half = _table->_size / 2; 422 423 // Let each partition be one page worth of buckets 424 size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(G1StringDedupEntry*)); 425 assert(table_half % partition_size == 0, "Invalid partition size"); 426 427 // Number of entries removed during the scan 428 uintx removed = 0; 429 430 for (;;) { 431 // Grab next partition to scan 432 size_t partition_begin = cl->claim_table_partition(partition_size); 433 size_t partition_end = partition_begin + partition_size; 434 if (partition_begin >= table_half) { 435 // End of table 436 break; 437 } 438 439 // Scan the partition followed by the sibling partition in the second half of the table 440 removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id); 441 removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id); 442 } 443 444 // Delayed update avoid contention on the table lock 445 if (removed > 0) { 446 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); 447 _table->_entries -= removed; 448 _entries_removed += removed; 449 } 450 } 451 452 uintx G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, 453 size_t partition_begin, 454 size_t partition_end, 455 uint worker_id) { 456 uintx removed = 0; 457 for (size_t bucket = partition_begin; bucket < partition_end; bucket++) { 458 G1StringDedupEntry** entry = _table->bucket(bucket); 459 while (*entry != NULL) { 460 oop* p = (oop*)(*entry)->obj_addr(); 461 if (cl->is_alive(*p)) { 462 cl->keep_alive(p); 463 if (cl->is_resizing()) { 464 // We are resizing the table, transfer entry to the new table 465 _table->transfer(entry, cl->resized_table()); 466 } else { 467 if (cl->is_rehashing()) { 468 // We are rehashing the table, rehash the entry but keep it 469 // in the table. We can't transfer entries into the new table 470 // at this point since we don't have exclusive access to all 471 // destination partitions. finish_rehash() will do a single 472 // threaded transfer of all entries. 473 typeArrayOop value = (typeArrayOop)*p; 474 bool latin1 = (*entry)->latin1(); 475 unsigned int hash = hash_code(value, latin1); 476 (*entry)->set_hash(hash); 477 } 478 479 // Move to next entry 480 entry = (*entry)->next_addr(); 481 } 482 } else { 483 // Not alive, remove entry from table 484 _table->remove(entry, worker_id); 485 removed++; 486 } 487 } 488 } 489 490 return removed; 491 } 492 493 G1StringDedupTable* G1StringDedupTable::prepare_rehash() { 494 if (!_table->_rehash_needed && !StringDeduplicationRehashALot) { 495 // Rehash not needed 496 return NULL; 497 } 498 499 // Update statistics 500 _rehash_count++; 501 502 // Compute new hash seed 503 _table->_hash_seed = AltHashing::compute_seed(); 504 505 // Allocate the new table, same size and hash seed 506 return new G1StringDedupTable(_table->_size, _table->_hash_seed); 507 } 508 509 void G1StringDedupTable::finish_rehash(G1StringDedupTable* rehashed_table) { 510 assert(rehashed_table != NULL, "Invalid table"); 511 512 // Move all newly rehashed entries into the correct buckets in the new table 513 for (size_t bucket = 0; bucket < _table->_size; bucket++) { 514 G1StringDedupEntry** entry = _table->bucket(bucket); 515 while (*entry != NULL) { 516 _table->transfer(entry, rehashed_table); 517 } 518 } 519 520 rehashed_table->_entries = _table->_entries; 521 522 // Free old table 523 delete _table; 524 525 // Install new table 526 _table = rehashed_table; 527 } 528 529 void G1StringDedupTable::verify() { 530 for (size_t bucket = 0; bucket < _table->_size; bucket++) { 531 // Verify entries 532 G1StringDedupEntry** entry = _table->bucket(bucket); 533 while (*entry != NULL) { 534 typeArrayOop value = (*entry)->obj(); 535 guarantee(value != NULL, "Object must not be NULL"); 536 guarantee(G1CollectedHeap::heap()->is_in_reserved(value), "Object must be on the heap"); 537 guarantee(!value->is_forwarded(), "Object must not be forwarded"); 538 guarantee(value->is_typeArray(), "Object must be a typeArrayOop"); 539 bool latin1 = (*entry)->latin1(); 540 unsigned int hash = hash_code(value, latin1); 541 guarantee((*entry)->hash() == hash, "Table entry has inorrect hash"); 542 guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index"); 543 entry = (*entry)->next_addr(); 544 } 545 546 // Verify that we do not have entries with identical oops or identical arrays. 547 // We only need to compare entries in the same bucket. If the same oop or an 548 // identical array has been inserted more than once into different/incorrect 549 // buckets the verification step above will catch that. 550 G1StringDedupEntry** entry1 = _table->bucket(bucket); 551 while (*entry1 != NULL) { 552 typeArrayOop value1 = (*entry1)->obj(); 553 bool latin1_1 = (*entry1)->latin1(); 554 G1StringDedupEntry** entry2 = (*entry1)->next_addr(); 555 while (*entry2 != NULL) { 556 typeArrayOop value2 = (*entry2)->obj(); 557 bool latin1_2 = (*entry2)->latin1(); 558 guarantee(latin1_1 != latin1_2 || !equals(value1, value2), "Table entries must not have identical arrays"); 559 entry2 = (*entry2)->next_addr(); 560 } 561 entry1 = (*entry1)->next_addr(); 562 } 563 } 564 } 565 566 void G1StringDedupTable::trim_entry_cache() { 567 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); 568 size_t max_cache_size = (size_t)(_table->_size * _max_cache_factor); 569 _entry_cache->trim(max_cache_size); 570 } 571 572 void G1StringDedupTable::print_statistics() { 573 Log(gc, stringdedup) log; 574 log.debug(" [Table]"); 575 log.debug(" [Memory Usage: " G1_STRDEDUP_BYTES_FORMAT_NS "]", 576 G1_STRDEDUP_BYTES_PARAM(_table->_size * sizeof(G1StringDedupEntry*) + (_table->_entries + _entry_cache->size()) * sizeof(G1StringDedupEntry))); 577 log.debug(" [Size: " SIZE_FORMAT ", Min: " SIZE_FORMAT ", Max: " SIZE_FORMAT "]", _table->_size, _min_size, _max_size); 578 log.debug(" [Entries: " UINTX_FORMAT ", Load: " G1_STRDEDUP_PERCENT_FORMAT_NS ", Cached: " UINTX_FORMAT ", Added: " UINTX_FORMAT ", Removed: " UINTX_FORMAT "]", 579 _table->_entries, (double)_table->_entries / (double)_table->_size * 100.0, _entry_cache->size(), _entries_added, _entries_removed); 580 log.debug(" [Resize Count: " UINTX_FORMAT ", Shrink Threshold: " UINTX_FORMAT "(" G1_STRDEDUP_PERCENT_FORMAT_NS "), Grow Threshold: " UINTX_FORMAT "(" G1_STRDEDUP_PERCENT_FORMAT_NS ")]", 581 _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0); 582 log.debug(" [Rehash Count: " UINTX_FORMAT ", Rehash Threshold: " UINTX_FORMAT ", Hash Seed: 0x%x]", _rehash_count, _rehash_threshold, _table->_hash_seed); 583 log.debug(" [Age Threshold: " UINTX_FORMAT "]", StringDeduplicationAgeThreshold); 584 }