1 /* 2 * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/altHashing.hpp" 27 #include "classfile/javaClasses.inline.hpp" 28 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" 29 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" 30 #include "gc_implementation/g1/g1StringDedupTable.hpp" 31 #include "memory/gcLocker.hpp" 32 #include "memory/padded.inline.hpp" 33 #include "oops/typeArrayOop.hpp" 34 #include "oops/oop.inline.hpp" 35 #include "runtime/mutexLocker.hpp" 36 37 // 38 // Freelist in the deduplication table entry cache. Links table 39 // entries together using their _next fields. 40 // 41 class G1StringDedupEntryFreeList : public CHeapObj<mtGC> { 42 private: 43 G1StringDedupEntry* _list; 44 size_t _length; 45 46 public: 47 G1StringDedupEntryFreeList() : 48 _list(NULL), 49 _length(0) { 50 } 51 52 void add(G1StringDedupEntry* entry) { 53 entry->set_next(_list); 54 _list = entry; 55 _length++; 56 } 57 58 G1StringDedupEntry* remove() { 59 G1StringDedupEntry* entry = _list; 60 if (entry != NULL) { 61 _list = entry->next(); 62 _length--; 63 } 64 return entry; 65 } 66 67 size_t length() { 68 return _length; 69 } 70 }; 71 72 // 73 // Cache of deduplication table entries. This cache provides fast allocation and 74 // reuse of table entries to lower the pressure on the underlying allocator. 75 // But more importantly, it provides fast/deferred freeing of table entries. This 76 // is important because freeing of table entries is done during stop-the-world 77 // phases and it is not uncommon for large number of entries to be freed at once. 78 // Tables entries that are freed during these phases are placed onto a freelist in 79 // the cache. The deduplication thread, which executes in a concurrent phase, will 80 // later reuse or free the underlying memory for these entries. 81 // 82 // The cache allows for single-threaded allocations and multi-threaded frees. 83 // Allocations are synchronized by StringDedupTable_lock as part of a table 84 // modification. 85 // 86 class G1StringDedupEntryCache : public CHeapObj<mtGC> { 87 private: 88 // One freelist per GC worker to allow lock less freeing of 89 // entries while doing a parallel scan of the table. Using 90 // PaddedEnd to avoid false sharing. 91 PaddedEnd<G1StringDedupEntryFreeList>* _lists; 92 size_t _nlists; 93 94 public: 95 G1StringDedupEntryCache(); 96 ~G1StringDedupEntryCache(); 97 98 // Get a table entry from the cache freelist, or allocate a new 99 // entry if the cache is empty. 100 G1StringDedupEntry* alloc(); 101 102 // Insert a table entry into the cache freelist. 103 void free(G1StringDedupEntry* entry, uint worker_id); 104 105 // Returns current number of entries in the cache. 106 size_t size(); 107 108 // If the cache has grown above the given max size, trim it down 109 // and deallocate the memory occupied by trimmed of entries. 110 void trim(size_t max_size); 111 }; 112 113 G1StringDedupEntryCache::G1StringDedupEntryCache() { 114 _nlists = MAX2(ParallelGCThreads, (size_t)1); 115 _lists = PaddedArray<G1StringDedupEntryFreeList, mtGC>::create_unfreeable((uint)_nlists); 116 } 117 118 G1StringDedupEntryCache::~G1StringDedupEntryCache() { 119 ShouldNotReachHere(); 120 } 121 122 G1StringDedupEntry* G1StringDedupEntryCache::alloc() { 123 for (size_t i = 0; i < _nlists; i++) { 124 G1StringDedupEntry* entry = _lists[i].remove(); 125 if (entry != NULL) { 126 return entry; 127 } 128 } 129 return new G1StringDedupEntry(); 130 } 131 132 void G1StringDedupEntryCache::free(G1StringDedupEntry* entry, uint worker_id) { 133 assert(entry->obj() != NULL, "Double free"); 134 assert(worker_id < _nlists, "Invalid worker id"); 135 entry->set_obj(NULL); 136 entry->set_hash(0); 137 _lists[worker_id].add(entry); 138 } 139 140 size_t G1StringDedupEntryCache::size() { 141 size_t size = 0; 142 for (size_t i = 0; i < _nlists; i++) { 143 size += _lists[i].length(); 144 } 145 return size; 146 } 147 148 void G1StringDedupEntryCache::trim(size_t max_size) { 149 size_t cache_size = 0; 150 for (size_t i = 0; i < _nlists; i++) { 151 G1StringDedupEntryFreeList* list = &_lists[i]; 152 cache_size += list->length(); 153 while (cache_size > max_size) { 154 G1StringDedupEntry* entry = list->remove(); 155 assert(entry != NULL, "Should not be null"); 156 cache_size--; 157 delete entry; 158 } 159 } 160 } 161 162 G1StringDedupTable* G1StringDedupTable::_table = NULL; 163 G1StringDedupEntryCache* G1StringDedupTable::_entry_cache = NULL; 164 165 const size_t G1StringDedupTable::_min_size = (1 << 10); // 1024 166 const size_t G1StringDedupTable::_max_size = (1 << 24); // 16777216 167 const double G1StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load 168 const double G1StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load 169 const double G1StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size 170 const uintx G1StringDedupTable::_rehash_multiple = 60; // Hash bucket has 60 times more collisions than expected 171 const uintx G1StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor); 172 173 uintx G1StringDedupTable::_entries_added = 0; 174 uintx G1StringDedupTable::_entries_removed = 0; 175 uintx G1StringDedupTable::_resize_count = 0; 176 uintx G1StringDedupTable::_rehash_count = 0; 177 178 G1StringDedupTable::G1StringDedupTable(size_t size, jint hash_seed) : 179 _size(size), 180 _entries(0), 181 _grow_threshold((uintx)(size * _grow_load_factor)), 182 _shrink_threshold((uintx)(size * _shrink_load_factor)), 183 _rehash_needed(false), 184 _hash_seed(hash_seed) { 185 assert(is_power_of_2(size), "Table size must be a power of 2"); 186 _buckets = NEW_C_HEAP_ARRAY(G1StringDedupEntry*, _size, mtGC); 187 memset(_buckets, 0, _size * sizeof(G1StringDedupEntry*)); 188 } 189 190 G1StringDedupTable::~G1StringDedupTable() { 191 FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets); 192 } 193 194 void G1StringDedupTable::create() { 195 assert(_table == NULL, "One string deduplication table allowed"); 196 _entry_cache = new G1StringDedupEntryCache(); 197 _table = new G1StringDedupTable(_min_size); 198 } 199 200 void G1StringDedupTable::add(typeArrayOop value, unsigned int hash, G1StringDedupEntry** list) { 201 G1StringDedupEntry* entry = _entry_cache->alloc(); 202 entry->set_obj(value); 203 entry->set_hash(hash); 204 entry->set_next(*list); 205 *list = entry; 206 _entries++; 207 } 208 209 void G1StringDedupTable::remove(G1StringDedupEntry** pentry, uint worker_id) { 210 G1StringDedupEntry* entry = *pentry; 211 *pentry = entry->next(); 212 _entry_cache->free(entry, worker_id); 213 } 214 215 void G1StringDedupTable::transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest) { 216 G1StringDedupEntry* entry = *pentry; 217 *pentry = entry->next(); 218 unsigned int hash = entry->hash(); 219 size_t index = dest->hash_to_index(hash); 220 G1StringDedupEntry** list = dest->bucket(index); 221 entry->set_next(*list); 222 *list = entry; 223 } 224 225 bool G1StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) { 226 return (value1 == value2 || 227 (value1->length() == value2->length() && 228 (!memcmp(value1->base(T_CHAR), 229 value2->base(T_CHAR), 230 value1->length() * sizeof(jchar))))); 231 } 232 233 typeArrayOop G1StringDedupTable::lookup(typeArrayOop value, unsigned int hash, 234 G1StringDedupEntry** list, uintx &count) { 235 for (G1StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) { 236 if (entry->hash() == hash) { 237 typeArrayOop existing_value = entry->obj(); 238 if (equals(value, existing_value)) { 239 // Match found 240 return existing_value; 241 } 242 } 243 count++; 244 } 245 246 // Not found 247 return NULL; 248 } 249 250 typeArrayOop G1StringDedupTable::lookup_or_add_inner(typeArrayOop value, unsigned int hash) { 251 size_t index = hash_to_index(hash); 252 G1StringDedupEntry** list = bucket(index); 253 uintx count = 0; 254 255 // Lookup in list 256 typeArrayOop existing_value = lookup(value, hash, list, count); 257 258 // Check if rehash is needed 259 if (count > _rehash_threshold) { 260 _rehash_needed = true; 261 } 262 263 if (existing_value == NULL) { 264 // Not found, add new entry 265 add(value, hash, list); 266 267 // Update statistics 268 _entries_added++; 269 } 270 271 return existing_value; 272 } 273 274 unsigned int G1StringDedupTable::hash_code(typeArrayOop value) { 275 unsigned int hash; 276 int length = value->length(); 277 const jchar* data = (jchar*)value->base(T_CHAR); 278 279 if (use_java_hash()) { 280 hash = java_lang_String::hash_code(data, length); 281 } else { 282 hash = AltHashing::murmur3_32(_table->_hash_seed, data, length); 283 } 284 285 return hash; 286 } 287 288 void G1StringDedupTable::deduplicate(oop java_string, G1StringDedupStat& stat) { 289 assert(java_lang_String::is_instance(java_string), "Must be a string"); 290 No_Safepoint_Verifier nsv; 291 292 stat.inc_inspected(); 293 294 typeArrayOop value = java_lang_String::value(java_string); 295 if (value == NULL) { 296 // String has no value 297 stat.inc_skipped(); 298 return; 299 } 300 301 unsigned int hash = 0; 302 303 if (use_java_hash()) { 304 // Get hash code from cache 305 hash = java_lang_String::hash(java_string); 306 } 307 308 if (hash == 0) { 309 // Compute hash 310 hash = hash_code(value); 311 stat.inc_hashed(); 312 } 313 314 if (use_java_hash() && hash != 0) { 315 // Store hash code in cache 316 java_lang_String::set_hash(java_string, hash); 317 } 318 319 typeArrayOop existing_value = lookup_or_add(value, hash); 320 if (existing_value == value) { 321 // Same value, already known 322 stat.inc_known(); 323 return; 324 } 325 326 // Get size of value array 327 uintx size_in_bytes = value->size() * HeapWordSize; 328 stat.inc_new(size_in_bytes); 329 330 if (existing_value != NULL) { 331 // Enqueue the reference to make sure it is kept alive. Concurrent mark might 332 // otherwise declare it dead if there are no other strong references to this object. 333 G1SATBCardTableModRefBS::enqueue(existing_value); 334 335 // Existing value found, deduplicate string 336 java_lang_String::set_value(java_string, existing_value); 337 338 if (G1CollectedHeap::heap()->is_in_young(value)) { 339 stat.inc_deduped_young(size_in_bytes); 340 } else { 341 stat.inc_deduped_old(size_in_bytes); 342 } 343 } 344 } 345 346 G1StringDedupTable* G1StringDedupTable::prepare_resize() { 347 size_t size = _table->_size; 348 349 // Check if the hashtable needs to be resized 350 if (_table->_entries > _table->_grow_threshold) { 351 // Grow table, double the size 352 size *= 2; 353 if (size > _max_size) { 354 // Too big, don't resize 355 return NULL; 356 } 357 } else if (_table->_entries < _table->_shrink_threshold) { 358 // Shrink table, half the size 359 size /= 2; 360 if (size < _min_size) { 361 // Too small, don't resize 362 return NULL; 363 } 364 } else if (StringDeduplicationResizeALot) { 365 // Force grow 366 size *= 2; 367 if (size > _max_size) { 368 // Too big, force shrink instead 369 size /= 4; 370 } 371 } else { 372 // Resize not needed 373 return NULL; 374 } 375 376 // Update statistics 377 _resize_count++; 378 379 // Allocate the new table. The new table will be populated by workers 380 // calling unlink_or_oops_do() and finally installed by finish_resize(). 381 return new G1StringDedupTable(size, _table->_hash_seed); 382 } 383 384 void G1StringDedupTable::finish_resize(G1StringDedupTable* resized_table) { 385 assert(resized_table != NULL, "Invalid table"); 386 387 resized_table->_entries = _table->_entries; 388 389 // Free old table 390 delete _table; 391 392 // Install new table 393 _table = resized_table; 394 } 395 396 void G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) { 397 // The table is divided into partitions to allow lock-less parallel processing by 398 // multiple worker threads. A worker thread first claims a partition, which ensures 399 // exclusive access to that part of the table, then continues to process it. To allow 400 // shrinking of the table in parallel we also need to make sure that the same worker 401 // thread processes all partitions where entries will hash to the same destination 402 // partition. Since the table size is always a power of two and we always shrink by 403 // dividing the table in half, we know that for a given partition there is only one 404 // other partition whoes entries will hash to the same destination partition. That 405 // other partition is always the sibling partition in the second half of the table. 406 // For example, if the table is divided into 8 partitions, the sibling of partition 0 407 // is partition 4, the sibling of partition 1 is partition 5, etc. 408 size_t table_half = _table->_size / 2; 409 410 // Let each partition be one page worth of buckets 411 size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(G1StringDedupEntry*)); 412 assert(table_half % partition_size == 0, "Invalid partition size"); 413 414 // Number of entries removed during the scan 415 uintx removed = 0; 416 417 for (;;) { 418 // Grab next partition to scan 419 size_t partition_begin = cl->claim_table_partition(partition_size); 420 size_t partition_end = partition_begin + partition_size; 421 if (partition_begin >= table_half) { 422 // End of table 423 break; 424 } 425 426 // Scan the partition followed by the sibling partition in the second half of the table 427 removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id); 428 removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id); 429 } 430 431 // Delayed update avoid contention on the table lock 432 if (removed > 0) { 433 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); 434 _table->_entries -= removed; 435 _entries_removed += removed; 436 } 437 } 438 439 uintx G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, 440 size_t partition_begin, 441 size_t partition_end, 442 uint worker_id) { 443 uintx removed = 0; 444 for (size_t bucket = partition_begin; bucket < partition_end; bucket++) { 445 G1StringDedupEntry** entry = _table->bucket(bucket); 446 while (*entry != NULL) { 447 oop* p = (oop*)(*entry)->obj_addr(); 448 if (cl->is_alive(*p)) { 449 cl->keep_alive(p); 450 if (cl->is_resizing()) { 451 // We are resizing the table, transfer entry to the new table 452 _table->transfer(entry, cl->resized_table()); 453 } else { 454 if (cl->is_rehashing()) { 455 // We are rehashing the table, rehash the entry but keep it 456 // in the table. We can't transfer entries into the new table 457 // at this point since we don't have exclusive access to all 458 // destination partitions. finish_rehash() will do a single 459 // threaded transfer of all entries. 460 typeArrayOop value = (typeArrayOop)*p; 461 unsigned int hash = hash_code(value); 462 (*entry)->set_hash(hash); 463 } 464 465 // Move to next entry 466 entry = (*entry)->next_addr(); 467 } 468 } else { 469 // Not alive, remove entry from table 470 _table->remove(entry, worker_id); 471 removed++; 472 } 473 } 474 } 475 476 return removed; 477 } 478 479 G1StringDedupTable* G1StringDedupTable::prepare_rehash() { 480 if (!_table->_rehash_needed && !StringDeduplicationRehashALot) { 481 // Rehash not needed 482 return NULL; 483 } 484 485 // Update statistics 486 _rehash_count++; 487 488 // Compute new hash seed 489 _table->_hash_seed = AltHashing::compute_seed(); 490 491 // Allocate the new table, same size and hash seed 492 return new G1StringDedupTable(_table->_size, _table->_hash_seed); 493 } 494 495 void G1StringDedupTable::finish_rehash(G1StringDedupTable* rehashed_table) { 496 assert(rehashed_table != NULL, "Invalid table"); 497 498 // Move all newly rehashed entries into the correct buckets in the new table 499 for (size_t bucket = 0; bucket < _table->_size; bucket++) { 500 G1StringDedupEntry** entry = _table->bucket(bucket); 501 while (*entry != NULL) { 502 _table->transfer(entry, rehashed_table); 503 } 504 } 505 506 rehashed_table->_entries = _table->_entries; 507 508 // Free old table 509 delete _table; 510 511 // Install new table 512 _table = rehashed_table; 513 } 514 515 void G1StringDedupTable::verify() { 516 for (size_t bucket = 0; bucket < _table->_size; bucket++) { 517 // Verify entries 518 G1StringDedupEntry** entry = _table->bucket(bucket); 519 while (*entry != NULL) { 520 typeArrayOop value = (*entry)->obj(); 521 guarantee(value != NULL, "Object must not be NULL"); 522 guarantee(G1CollectedHeap::heap()->is_in_reserved(value), "Object must be on the heap"); 523 guarantee(!value->is_forwarded(), "Object must not be forwarded"); 524 guarantee(value->is_typeArray(), "Object must be a typeArrayOop"); 525 unsigned int hash = hash_code(value); 526 guarantee((*entry)->hash() == hash, "Table entry has inorrect hash"); 527 guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index"); 528 entry = (*entry)->next_addr(); 529 } 530 531 // Verify that we do not have entries with identical oops or identical arrays. 532 // We only need to compare entries in the same bucket. If the same oop or an 533 // identical array has been inserted more than once into different/incorrect 534 // buckets the verification step above will catch that. 535 G1StringDedupEntry** entry1 = _table->bucket(bucket); 536 while (*entry1 != NULL) { 537 typeArrayOop value1 = (*entry1)->obj(); 538 G1StringDedupEntry** entry2 = (*entry1)->next_addr(); 539 while (*entry2 != NULL) { 540 typeArrayOop value2 = (*entry2)->obj(); 541 guarantee(!equals(value1, value2), "Table entries must not have identical arrays"); 542 entry2 = (*entry2)->next_addr(); 543 } 544 entry1 = (*entry1)->next_addr(); 545 } 546 } 547 } 548 549 void G1StringDedupTable::trim_entry_cache() { 550 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); 551 size_t max_cache_size = (size_t)(_table->_size * _max_cache_factor); 552 _entry_cache->trim(max_cache_size); 553 } 554 555 void G1StringDedupTable::print_statistics(outputStream* st) { 556 st->print_cr( 557 " [Table]\n" 558 " [Memory Usage: "G1_STRDEDUP_BYTES_FORMAT_NS"]\n" 559 " [Size: "SIZE_FORMAT", Min: "SIZE_FORMAT", Max: "SIZE_FORMAT"]\n" 560 " [Entries: "UINTX_FORMAT", Load: "G1_STRDEDUP_PERCENT_FORMAT_NS", Cached: " UINTX_FORMAT ", Added: "UINTX_FORMAT", Removed: "UINTX_FORMAT"]\n" 561 " [Resize Count: "UINTX_FORMAT", Shrink Threshold: "UINTX_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT_NS"), Grow Threshold: "UINTX_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT_NS")]\n" 562 " [Rehash Count: "UINTX_FORMAT", Rehash Threshold: "UINTX_FORMAT", Hash Seed: 0x%x]\n" 563 " [Age Threshold: "UINTX_FORMAT"]", 564 G1_STRDEDUP_BYTES_PARAM(_table->_size * sizeof(G1StringDedupEntry*) + (_table->_entries + _entry_cache->size()) * sizeof(G1StringDedupEntry)), 565 _table->_size, _min_size, _max_size, 566 _table->_entries, (double)_table->_entries / (double)_table->_size * 100.0, _entry_cache->size(), _entries_added, _entries_removed, 567 _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0, 568 _rehash_count, _rehash_threshold, _table->_hash_seed, 569 StringDeduplicationAgeThreshold); 570 }