8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/altHashing.hpp"
  27 #include "classfile/javaClasses.inline.hpp"
  28 #include "gc/g1/g1BarrierSet.hpp"
  29 #include "gc/g1/g1CollectedHeap.inline.hpp"
  30 #include "gc/g1/g1StringDedup.hpp"
  31 #include "gc/g1/g1StringDedupTable.hpp"
  32 #include "logging/log.hpp"
  33 #include "memory/padded.inline.hpp"

  34 #include "oops/arrayOop.inline.hpp"
  35 #include "oops/oop.inline.hpp"
  36 #include "oops/typeArrayOop.hpp"
  37 #include "runtime/mutexLocker.hpp"
  38 #include "runtime/safepointVerifiers.hpp"
  39 
  40 //
  41 // List of deduplication table entries. Links table
  42 // entries together using their _next fields.
  43 //
  44 class G1StringDedupEntryList : public CHeapObj<mtGC> {
  45 private:
  46   G1StringDedupEntry* _list;
  47   size_t              _length;
  48 
  49 public:
  50   G1StringDedupEntryList() :
  51     _list(NULL),
  52     _length(0) {
  53   }
  54 
  55   void add(G1StringDedupEntry* entry) {
  56     entry->set_next(_list);
  57     _list = entry;
  58     _length++;
  59   }
  60 
  61   G1StringDedupEntry* remove() {
  62     G1StringDedupEntry* entry = _list;
  63     if (entry != NULL) {
  64       _list = entry->next();
  65       _length--;
  66     }
  67     return entry;
  68   }
  69 
  70   G1StringDedupEntry* remove_all() {
  71     G1StringDedupEntry* list = _list;
  72     _list = NULL;
  73     return list;
  74   }
  75 
  76   size_t length() {
  77     return _length;
  78   }
  79 };
  80 
  81 //
  82 // Cache of deduplication table entries. This cache provides fast allocation and
  83 // reuse of table entries to lower the pressure on the underlying allocator.
  84 // But more importantly, it provides fast/deferred freeing of table entries. This
  85 // is important because freeing of table entries is done during stop-the-world
  86 // phases and it is not uncommon for large number of entries to be freed at once.
  87 // Tables entries that are freed during these phases are placed onto a freelist in
  88 // the cache. The deduplication thread, which executes in a concurrent phase, will
  89 // later reuse or free the underlying memory for these entries.
  90 //
  91 // The cache allows for single-threaded allocations and multi-threaded frees.
  92 // Allocations are synchronized by StringDedupTable_lock as part of a table
  93 // modification.
  94 //
  95 class G1StringDedupEntryCache : public CHeapObj<mtGC> {
  96 private:
  97   // One cache/overflow list per GC worker to allow lock less freeing of
  98   // entries while doing a parallel scan of the table. Using PaddedEnd to
  99   // avoid false sharing.
 100   size_t                             _nlists;
 101   size_t                             _max_list_length;
 102   PaddedEnd<G1StringDedupEntryList>* _cached;
 103   PaddedEnd<G1StringDedupEntryList>* _overflowed;
 104 
 105 public:
 106   G1StringDedupEntryCache(size_t max_size);
 107   ~G1StringDedupEntryCache();
 108 
 109   // Set max number of table entries to cache.
 110   void set_max_size(size_t max_size);
 111 
 112   // Get a table entry from the cache, or allocate a new entry if the cache is empty.
 113   G1StringDedupEntry* alloc();
 114 
 115   // Insert a table entry into the cache.
 116   void free(G1StringDedupEntry* entry, uint worker_id);
 117 
 118   // Returns current number of entries in the cache.
 119   size_t size();
 120 
 121   // Deletes overflowed entries.
 122   void delete_overflowed();
 123 };
 124 
 125 G1StringDedupEntryCache::G1StringDedupEntryCache(size_t max_size) :
 126   _nlists(ParallelGCThreads),
 127   _max_list_length(0),
 128   _cached(PaddedArray<G1StringDedupEntryList, mtGC>::create_unfreeable((uint)_nlists)),
 129   _overflowed(PaddedArray<G1StringDedupEntryList, mtGC>::create_unfreeable((uint)_nlists)) {
 130   set_max_size(max_size);
 131 }
 132 
 133 G1StringDedupEntryCache::~G1StringDedupEntryCache() {
 134   ShouldNotReachHere();
 135 }
 136 
 137 void G1StringDedupEntryCache::set_max_size(size_t size) {
 138   _max_list_length = size / _nlists;
 139 }
 140 
 141 G1StringDedupEntry* G1StringDedupEntryCache::alloc() {
 142   for (size_t i = 0; i < _nlists; i++) {
 143     G1StringDedupEntry* entry = _cached[i].remove();
 144     if (entry != NULL) {
 145       return entry;
 146     }
 147   }
 148   return new G1StringDedupEntry();
 149 }
 150 
 151 void G1StringDedupEntryCache::free(G1StringDedupEntry* entry, uint worker_id) {
 152   assert(entry->obj() != NULL, "Double free");
 153   assert(worker_id < _nlists, "Invalid worker id");
 154 
 155   entry->set_obj(NULL);
 156   entry->set_hash(0);
 157 
 158   if (_cached[worker_id].length() < _max_list_length) {
 159     // Cache is not full
 160     _cached[worker_id].add(entry);
 161   } else {
 162     // Cache is full, add to overflow list for later deletion
 163     _overflowed[worker_id].add(entry);
 164   }
 165 }
 166 
 167 size_t G1StringDedupEntryCache::size() {
 168   size_t size = 0;
 169   for (size_t i = 0; i < _nlists; i++) {
 170     size += _cached[i].length();
 171   }
 172   return size;
 173 }
 174 
 175 void G1StringDedupEntryCache::delete_overflowed() {
 176   double start = os::elapsedTime();
 177   uintx count = 0;
 178 
 179   for (size_t i = 0; i < _nlists; i++) {
 180     G1StringDedupEntry* entry;
 181 
 182     {
 183       // The overflow list can be modified during safepoints, therefore
 184       // we temporarily join the suspendible thread set while removing
 185       // all entries from the list.
 186       SuspendibleThreadSetJoiner sts_join;
 187       entry = _overflowed[i].remove_all();
 188     }
 189 
 190     // Delete all entries
 191     while (entry != NULL) {
 192       G1StringDedupEntry* next = entry->next();
 193       delete entry;
 194       entry = next;
 195       count++;
 196     }
 197   }
 198 
 199   double end = os::elapsedTime();
 200   log_trace(gc, stringdedup)("Deleted " UINTX_FORMAT " entries, " G1_STRDEDUP_TIME_FORMAT_MS,
 201                              count, G1_STRDEDUP_TIME_PARAM_MS(end - start));
 202 }
 203 
 204 G1StringDedupTable*      G1StringDedupTable::_table = NULL;
 205 G1StringDedupEntryCache* G1StringDedupTable::_entry_cache = NULL;
 206 
 207 const size_t             G1StringDedupTable::_min_size = (1 << 10);   // 1024
 208 const size_t             G1StringDedupTable::_max_size = (1 << 24);   // 16777216
 209 const double             G1StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load
 210 const double             G1StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load
 211 const double             G1StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size
 212 const uintx              G1StringDedupTable::_rehash_multiple = 60;   // Hash bucket has 60 times more collisions than expected
 213 const uintx              G1StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor);
 214 
 215 uintx                    G1StringDedupTable::_entries_added = 0;
 216 uintx                    G1StringDedupTable::_entries_removed = 0;
 217 uintx                    G1StringDedupTable::_resize_count = 0;
 218 uintx                    G1StringDedupTable::_rehash_count = 0;




 219 
 220 G1StringDedupTable::G1StringDedupTable(size_t size, jint hash_seed) :
 221   _size(size),
 222   _entries(0),
 223   _grow_threshold((uintx)(size * _grow_load_factor)),
 224   _shrink_threshold((uintx)(size * _shrink_load_factor)),
 225   _rehash_needed(false),
 226   _hash_seed(hash_seed) {
 227   assert(is_power_of_2(size), "Table size must be a power of 2");
 228   _buckets = NEW_C_HEAP_ARRAY(G1StringDedupEntry*, _size, mtGC);
 229   memset(_buckets, 0, _size * sizeof(G1StringDedupEntry*));
 230 }
 231 
 232 G1StringDedupTable::~G1StringDedupTable() {
 233   FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets);
 234 }
 235 
 236 void G1StringDedupTable::create() {
 237   assert(_table == NULL, "One string deduplication table allowed");
 238   _entry_cache = new G1StringDedupEntryCache(_min_size * _max_cache_factor);
 239   _table = new G1StringDedupTable(_min_size);
 240 }
 241 
 242 void G1StringDedupTable::add(typeArrayOop value, bool latin1, unsigned int hash, G1StringDedupEntry** list) {
 243   G1StringDedupEntry* entry = _entry_cache->alloc();
 244   entry->set_obj(value);
 245   entry->set_hash(hash);
 246   entry->set_latin1(latin1);
 247   entry->set_next(*list);
 248   *list = entry;
 249   _entries++;
 250 }
 251 
 252 void G1StringDedupTable::remove(G1StringDedupEntry** pentry, uint worker_id) {
 253   G1StringDedupEntry* entry = *pentry;
 254   *pentry = entry->next();
 255   _entry_cache->free(entry, worker_id);
 256 }
 257 
 258 void G1StringDedupTable::transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest) {
 259   G1StringDedupEntry* entry = *pentry;
 260   *pentry = entry->next();
 261   unsigned int hash = entry->hash();
 262   size_t index = dest->hash_to_index(hash);
 263   G1StringDedupEntry** list = dest->bucket(index);
 264   entry->set_next(*list);
 265   *list = entry;
 266 }
 267 
 268 bool G1StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) {
 269   return (value1 == value2 ||
 270           (value1->length() == value2->length() &&
 271            (!memcmp(value1->base(T_BYTE),
 272                     value2->base(T_BYTE),
 273                     value1->length() * sizeof(jbyte)))));
 274 }
 275 
 276 typeArrayOop G1StringDedupTable::lookup(typeArrayOop value, bool latin1, unsigned int hash,
 277                                         G1StringDedupEntry** list, uintx &count) {
 278   for (G1StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) {
 279     if (entry->hash() == hash && entry->latin1() == latin1) {
 280       typeArrayOop existing_value = entry->obj();
 281       if (equals(value, existing_value)) {
 282         // Match found
 283         return existing_value;



 284       }
 285     }
 286     count++;
 287   }
 288 
 289   // Not found
 290   return NULL;
 291 }
 292 
 293 typeArrayOop G1StringDedupTable::lookup_or_add_inner(typeArrayOop value, bool latin1, unsigned int hash) {
 294   size_t index = hash_to_index(hash);
 295   G1StringDedupEntry** list = bucket(index);
 296   uintx count = 0;
 297 
 298   // Lookup in list
 299   typeArrayOop existing_value = lookup(value, latin1, hash, list, count);
 300 
 301   // Check if rehash is needed
 302   if (count > _rehash_threshold) {
 303     _rehash_needed = true;
 304   }
 305 
 306   if (existing_value == NULL) {
 307     // Not found, add new entry
 308     add(value, latin1, hash, list);
 309 
 310     // Update statistics
 311     _entries_added++;
 312   }
 313 
 314   return existing_value;
 315 }
 316 
 317 unsigned int G1StringDedupTable::hash_code(typeArrayOop value, bool latin1) {
 318   unsigned int hash;
 319   int length = value->length();
 320   if (latin1) {
 321     const jbyte* data = (jbyte*)value->base(T_BYTE);
 322     if (use_java_hash()) {
 323       hash = java_lang_String::hash_code(data, length);
 324     } else {
 325       hash = AltHashing::murmur3_32(_table->_hash_seed, data, length);
 326     }
 327   } else {
 328     length /= sizeof(jchar) / sizeof(jbyte); // Convert number of bytes to number of chars
 329     const jchar* data = (jchar*)value->base(T_CHAR);
 330     if (use_java_hash()) {
 331       hash = java_lang_String::hash_code(data, length);
 332     } else {
 333       hash = AltHashing::murmur3_32(_table->_hash_seed, data, length);
 334     }
 335   }
 336 
 337   return hash;
 338 }
 339 
 340 void G1StringDedupTable::deduplicate(oop java_string, G1StringDedupStat& stat) {
 341   assert(java_lang_String::is_instance(java_string), "Must be a string");
 342   NoSafepointVerifier nsv;
 343 
 344   stat.inc_inspected();
 345 
 346   typeArrayOop value = java_lang_String::value(java_string);
 347   if (value == NULL) {
 348     // String has no value
 349     stat.inc_skipped();
 350     return;
 351   }
 352 
 353   bool latin1 = java_lang_String::is_latin1(java_string);
 354   unsigned int hash = 0;
 355 
 356   if (use_java_hash()) {
 357     // Get hash code from cache
 358     hash = java_lang_String::hash(java_string);
 359   }
 360 
 361   if (hash == 0) {
 362     // Compute hash
 363     hash = hash_code(value, latin1);
 364     stat.inc_hashed();
 365 
 366     if (use_java_hash() && hash != 0) {
 367       // Store hash code in cache
 368       java_lang_String::set_hash(java_string, hash);
 369     }
 370   }
 371 
 372   typeArrayOop existing_value = lookup_or_add(value, latin1, hash);
 373   if (existing_value == value) {
 374     // Same value, already known
 375     stat.inc_known();
 376     return;
 377   }
 378 
 379   // Get size of value array
 380   uintx size_in_bytes = value->size() * HeapWordSize;
 381   stat.inc_new(size_in_bytes);
 382 
 383   if (existing_value != NULL) {
 384     // Enqueue the reference to make sure it is kept alive. Concurrent mark might
 385     // otherwise declare it dead if there are no other strong references to this object.
 386     G1BarrierSet::enqueue(existing_value);
 387 
 388     // Existing value found, deduplicate string
 389     java_lang_String::set_value(java_string, existing_value);
 390 
 391     if (G1CollectedHeap::heap()->is_in_young(value)) {
 392       stat.inc_deduped_young(size_in_bytes);
 393     } else {
 394       stat.inc_deduped_old(size_in_bytes);
 395     }
 396   }
 397 }
 398 
 399 G1StringDedupTable* G1StringDedupTable::prepare_resize() {








 400   size_t size = _table->_size;
 401 
 402   // Check if the hashtable needs to be resized
 403   if (_table->_entries > _table->_grow_threshold) {
 404     // Grow table, double the size
 405     size *= 2;
 406     if (size > _max_size) {
 407       // Too big, don't resize
 408       return NULL;
 409     }
 410   } else if (_table->_entries < _table->_shrink_threshold) {
 411     // Shrink table, half the size
 412     size /= 2;
 413     if (size < _min_size) {
 414       // Too small, don't resize
 415       return NULL;
 416     }
 417   } else if (StringDeduplicationResizeALot) {
 418     // Force grow
 419     size *= 2;
 420     if (size > _max_size) {
 421       // Too big, force shrink instead
 422       size /= 4;
 423     }
 424   } else {
 425     // Resize not needed
 426     return NULL;
 427   }
 428 
 429   // Update statistics
 430   _resize_count++;
 431 
 432   // Update max cache size
 433   _entry_cache->set_max_size(size * _max_cache_factor);
 434 
 435   // Allocate the new table. The new table will be populated by workers
 436   // calling unlink_or_oops_do() and finally installed by finish_resize().
 437   return new G1StringDedupTable(size, _table->_hash_seed);
 438 }
 439 
 440 void G1StringDedupTable::finish_resize(G1StringDedupTable* resized_table) {
 441   assert(resized_table != NULL, "Invalid table");
 442 
 443   resized_table->_entries = _table->_entries;
 444 
 445   // Free old table
 446   delete _table;
 447 
 448   // Install new table
 449   _table = resized_table;
 450 }
 451 
 452 void G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) {
 453   // The table is divided into partitions to allow lock-less parallel processing by
 454   // multiple worker threads. A worker thread first claims a partition, which ensures
 455   // exclusive access to that part of the table, then continues to process it. To allow
 456   // shrinking of the table in parallel we also need to make sure that the same worker
 457   // thread processes all partitions where entries will hash to the same destination
 458   // partition. Since the table size is always a power of two and we always shrink by
 459   // dividing the table in half, we know that for a given partition there is only one
 460   // other partition whoes entries will hash to the same destination partition. That
 461   // other partition is always the sibling partition in the second half of the table.
 462   // For example, if the table is divided into 8 partitions, the sibling of partition 0
 463   // is partition 4, the sibling of partition 1 is partition 5, etc.
 464   size_t table_half = _table->_size / 2;
 465 
 466   // Let each partition be one page worth of buckets
 467   size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(G1StringDedupEntry*));
 468   assert(table_half % partition_size == 0, "Invalid partition size");
 469 
 470   // Number of entries removed during the scan
 471   uintx removed = 0;
 472 
 473   for (;;) {
 474     // Grab next partition to scan
 475     size_t partition_begin = cl->claim_table_partition(partition_size);
 476     size_t partition_end = partition_begin + partition_size;
 477     if (partition_begin >= table_half) {
 478       // End of table
 479       break;
 480     }
 481 
 482     // Scan the partition followed by the sibling partition in the second half of the table
 483     removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id);
 484     removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id);
 485   }
 486 
 487   // Delayed update to avoid contention on the table lock
 488   if (removed > 0) {
 489     MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag);
 490     _table->_entries -= removed;
 491     _entries_removed += removed;
 492   }
 493 }
 494 
 495 uintx G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl,
 496                                             size_t partition_begin,
 497                                             size_t partition_end,
 498                                             uint worker_id) {
 499   uintx removed = 0;
 500   for (size_t bucket = partition_begin; bucket < partition_end; bucket++) {
 501     G1StringDedupEntry** entry = _table->bucket(bucket);
 502     while (*entry != NULL) {
 503       oop* p = (oop*)(*entry)->obj_addr();
 504       if (cl->is_alive(*p)) {
 505         cl->keep_alive(p);
 506         if (cl->is_resizing()) {
 507           // We are resizing the table, transfer entry to the new table
 508           _table->transfer(entry, cl->resized_table());
 509         } else {
 510           if (cl->is_rehashing()) {
 511             // We are rehashing the table, rehash the entry but keep it
 512             // in the table. We can't transfer entries into the new table
 513             // at this point since we don't have exclusive access to all
 514             // destination partitions. finish_rehash() will do a single
 515             // threaded transfer of all entries.
 516             typeArrayOop value = (typeArrayOop)*p;
 517             bool latin1 = (*entry)->latin1();
 518             unsigned int hash = hash_code(value, latin1);
 519             (*entry)->set_hash(hash);
 520           }
 521 
 522           // Move to next entry
 523           entry = (*entry)->next_addr();
 524         }
 525       } else {
 526         // Not alive, remove entry from table
 527         _table->remove(entry, worker_id);
 528         removed++;
 529       }
 530     }
 531   }
 532 
 533   return removed;
 534 }
 535 
 536 G1StringDedupTable* G1StringDedupTable::prepare_rehash() {



























 537   if (!_table->_rehash_needed && !StringDeduplicationRehashALot) {
 538     // Rehash not needed
 539     return NULL;
 540   }
 541 
 542   // Update statistics
 543   _rehash_count++;
 544 
 545   // Compute new hash seed
 546   _table->_hash_seed = AltHashing::compute_seed();
 547 
 548   // Allocate the new table, same size and hash seed
 549   return new G1StringDedupTable(_table->_size, _table->_hash_seed);
 550 }
 551 
 552 void G1StringDedupTable::finish_rehash(G1StringDedupTable* rehashed_table) {
 553   assert(rehashed_table != NULL, "Invalid table");
 554 
 555   // Move all newly rehashed entries into the correct buckets in the new table
 556   for (size_t bucket = 0; bucket < _table->_size; bucket++) {
 557     G1StringDedupEntry** entry = _table->bucket(bucket);
 558     while (*entry != NULL) {
 559       _table->transfer(entry, rehashed_table);
 560     }
 561   }
 562 
 563   rehashed_table->_entries = _table->_entries;
 564 
 565   // Free old table
 566   delete _table;
 567 
 568   // Install new table
 569   _table = rehashed_table;
 570 }
 571 
 572 void G1StringDedupTable::verify() {




 573   for (size_t bucket = 0; bucket < _table->_size; bucket++) {
 574     // Verify entries
 575     G1StringDedupEntry** entry = _table->bucket(bucket);
 576     while (*entry != NULL) {
 577       typeArrayOop value = (*entry)->obj();
 578       guarantee(value != NULL, "Object must not be NULL");
 579       guarantee(G1CollectedHeap::heap()->is_in_reserved(value), "Object must be on the heap");
 580       guarantee(!value->is_forwarded(), "Object must not be forwarded");
 581       guarantee(value->is_typeArray(), "Object must be a typeArrayOop");
 582       bool latin1 = (*entry)->latin1();
 583       unsigned int hash = hash_code(value, latin1);
 584       guarantee((*entry)->hash() == hash, "Table entry has inorrect hash");
 585       guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index");
 586       entry = (*entry)->next_addr();
 587     }
 588 
 589     // Verify that we do not have entries with identical oops or identical arrays.
 590     // We only need to compare entries in the same bucket. If the same oop or an
 591     // identical array has been inserted more than once into different/incorrect
 592     // buckets the verification step above will catch that.
 593     G1StringDedupEntry** entry1 = _table->bucket(bucket);
 594     while (*entry1 != NULL) {
 595       typeArrayOop value1 = (*entry1)->obj();
 596       bool latin1_1 = (*entry1)->latin1();
 597       G1StringDedupEntry** entry2 = (*entry1)->next_addr();
 598       while (*entry2 != NULL) {
 599         typeArrayOop value2 = (*entry2)->obj();
 600         bool latin1_2 = (*entry2)->latin1();
 601         guarantee(latin1_1 != latin1_2 || !equals(value1, value2), "Table entries must not have identical arrays");
 602         entry2 = (*entry2)->next_addr();
 603       }
 604       entry1 = (*entry1)->next_addr();
 605     }
 606   }
 607 }
 608 
 609 void G1StringDedupTable::clean_entry_cache() {
 610   _entry_cache->delete_overflowed();
 611 }
 612 
 613 void G1StringDedupTable::print_statistics() {
 614   Log(gc, stringdedup) log;
 615   log.debug("  Table");
 616   log.debug("    Memory Usage: " G1_STRDEDUP_BYTES_FORMAT_NS,
 617             G1_STRDEDUP_BYTES_PARAM(_table->_size * sizeof(G1StringDedupEntry*) + (_table->_entries + _entry_cache->size()) * sizeof(G1StringDedupEntry)));
 618   log.debug("    Size: " SIZE_FORMAT ", Min: " SIZE_FORMAT ", Max: " SIZE_FORMAT, _table->_size, _min_size, _max_size);
 619   log.debug("    Entries: " UINTX_FORMAT ", Load: " G1_STRDEDUP_PERCENT_FORMAT_NS ", Cached: " UINTX_FORMAT ", Added: " UINTX_FORMAT ", Removed: " UINTX_FORMAT,
 620             _table->_entries, percent_of(_table->_entries, _table->_size), _entry_cache->size(), _entries_added, _entries_removed);
 621   log.debug("    Resize Count: " UINTX_FORMAT ", Shrink Threshold: " UINTX_FORMAT "(" G1_STRDEDUP_PERCENT_FORMAT_NS "), Grow Threshold: " UINTX_FORMAT "(" G1_STRDEDUP_PERCENT_FORMAT_NS ")",
 622             _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0);
 623   log.debug("    Rehash Count: " UINTX_FORMAT ", Rehash Threshold: " UINTX_FORMAT ", Hash Seed: 0x%x", _rehash_count, _rehash_threshold, _table->_hash_seed);
 624   log.debug("    Age Threshold: " UINTX_FORMAT, StringDeduplicationAgeThreshold);
 625 }

   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/altHashing.hpp"
  27 #include "classfile/javaClasses.inline.hpp"
  28 #include "gc/shared/stringdedup/stringDedup.hpp"
  29 #include "gc/shared/stringdedup/stringDedupTable.hpp"
  30 #include "gc/shared/suspendibleThreadSet.hpp"

  31 #include "logging/log.hpp"
  32 #include "memory/padded.inline.hpp"
  33 #include "oops/access.inline.hpp"
  34 #include "oops/arrayOop.inline.hpp"
  35 #include "oops/oop.inline.hpp"
  36 #include "oops/typeArrayOop.hpp"
  37 #include "runtime/mutexLocker.hpp"
  38 #include "runtime/safepointVerifiers.hpp"
  39 
  40 //
  41 // List of deduplication table entries. Links table
  42 // entries together using their _next fields.
  43 //
  44 class StringDedupEntryList : public CHeapObj<mtGC> {
  45 private:
  46   StringDedupEntry*   _list;
  47   size_t              _length;
  48 
  49 public:
  50   StringDedupEntryList() :
  51     _list(NULL),
  52     _length(0) {
  53   }
  54 
  55   void add(StringDedupEntry* entry) {
  56     entry->set_next(_list);
  57     _list = entry;
  58     _length++;
  59   }
  60 
  61   StringDedupEntry* remove() {
  62     StringDedupEntry* entry = _list;
  63     if (entry != NULL) {
  64       _list = entry->next();
  65       _length--;
  66     }
  67     return entry;
  68   }
  69 
  70   StringDedupEntry* remove_all() {
  71     StringDedupEntry* list = _list;
  72     _list = NULL;
  73     return list;
  74   }
  75 
  76   size_t length() {
  77     return _length;
  78   }
  79 };
  80 
  81 //
  82 // Cache of deduplication table entries. This cache provides fast allocation and
  83 // reuse of table entries to lower the pressure on the underlying allocator.
  84 // But more importantly, it provides fast/deferred freeing of table entries. This
  85 // is important because freeing of table entries is done during stop-the-world
  86 // phases and it is not uncommon for large number of entries to be freed at once.
  87 // Tables entries that are freed during these phases are placed onto a freelist in
  88 // the cache. The deduplication thread, which executes in a concurrent phase, will
  89 // later reuse or free the underlying memory for these entries.
  90 //
  91 // The cache allows for single-threaded allocations and multi-threaded frees.
  92 // Allocations are synchronized by StringDedupTable_lock as part of a table
  93 // modification.
  94 //
  95 class StringDedupEntryCache : public CHeapObj<mtGC> {
  96 private:
  97   // One cache/overflow list per GC worker to allow lock less freeing of
  98   // entries while doing a parallel scan of the table. Using PaddedEnd to
  99   // avoid false sharing.
 100   size_t                             _nlists;
 101   size_t                             _max_list_length;
 102   PaddedEnd<StringDedupEntryList>*   _cached;
 103   PaddedEnd<StringDedupEntryList>*   _overflowed;
 104 
 105 public:
 106   StringDedupEntryCache(size_t max_size);
 107   ~StringDedupEntryCache();
 108 
 109   // Set max number of table entries to cache.
 110   void set_max_size(size_t max_size);
 111 
 112   // Get a table entry from the cache, or allocate a new entry if the cache is empty.
 113   StringDedupEntry* alloc();
 114 
 115   // Insert a table entry into the cache.
 116   void free(StringDedupEntry* entry, uint worker_id);
 117 
 118   // Returns current number of entries in the cache.
 119   size_t size();
 120 
 121   // Deletes overflowed entries.
 122   void delete_overflowed();
 123 };
 124 
 125 StringDedupEntryCache::StringDedupEntryCache(size_t max_size) :
 126   _nlists(ParallelGCThreads),
 127   _max_list_length(0),
 128   _cached(PaddedArray<StringDedupEntryList, mtGC>::create_unfreeable((uint)_nlists)),
 129   _overflowed(PaddedArray<StringDedupEntryList, mtGC>::create_unfreeable((uint)_nlists)) {
 130   set_max_size(max_size);
 131 }
 132 
 133 StringDedupEntryCache::~StringDedupEntryCache() {
 134   ShouldNotReachHere();
 135 }
 136 
 137 void StringDedupEntryCache::set_max_size(size_t size) {
 138   _max_list_length = size / _nlists;
 139 }
 140 
 141 StringDedupEntry* StringDedupEntryCache::alloc() {
 142   for (size_t i = 0; i < _nlists; i++) {
 143     StringDedupEntry* entry = _cached[i].remove();
 144     if (entry != NULL) {
 145       return entry;
 146     }
 147   }
 148   return new StringDedupEntry();
 149 }
 150 
 151 void StringDedupEntryCache::free(StringDedupEntry* entry, uint worker_id) {
 152   assert(entry->obj() != NULL, "Double free");
 153   assert(worker_id < _nlists, "Invalid worker id");
 154 
 155   entry->set_obj(NULL);
 156   entry->set_hash(0);
 157 
 158   if (_cached[worker_id].length() < _max_list_length) {
 159     // Cache is not full
 160     _cached[worker_id].add(entry);
 161   } else {
 162     // Cache is full, add to overflow list for later deletion
 163     _overflowed[worker_id].add(entry);
 164   }
 165 }
 166 
 167 size_t StringDedupEntryCache::size() {
 168   size_t size = 0;
 169   for (size_t i = 0; i < _nlists; i++) {
 170     size += _cached[i].length();
 171   }
 172   return size;
 173 }
 174 
 175 void StringDedupEntryCache::delete_overflowed() {
 176   double start = os::elapsedTime();
 177   uintx count = 0;
 178 
 179   for (size_t i = 0; i < _nlists; i++) {
 180     StringDedupEntry* entry;
 181 
 182     {
 183       // The overflow list can be modified during safepoints, therefore
 184       // we temporarily join the suspendible thread set while removing
 185       // all entries from the list.
 186       SuspendibleThreadSetJoiner sts_join;
 187       entry = _overflowed[i].remove_all();
 188     }
 189 
 190     // Delete all entries
 191     while (entry != NULL) {
 192       StringDedupEntry* next = entry->next();
 193       delete entry;
 194       entry = next;
 195       count++;
 196     }
 197   }
 198 
 199   double end = os::elapsedTime();
 200   log_trace(gc, stringdedup)("Deleted " UINTX_FORMAT " entries, " STRDEDUP_TIME_FORMAT_MS,
 201                              count, STRDEDUP_TIME_PARAM_MS(end - start));
 202 }
 203 
 204 StringDedupTable*        StringDedupTable::_table = NULL;
 205 StringDedupEntryCache*   StringDedupTable::_entry_cache = NULL;
 206 
 207 const size_t             StringDedupTable::_min_size = (1 << 10);   // 1024
 208 const size_t             StringDedupTable::_max_size = (1 << 24);   // 16777216
 209 const double             StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load
 210 const double             StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load
 211 const double             StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size
 212 const uintx              StringDedupTable::_rehash_multiple = 60;   // Hash bucket has 60 times more collisions than expected
 213 const uintx              StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor);
 214 
 215 uintx                    StringDedupTable::_entries_added = 0;
 216 uintx                    StringDedupTable::_entries_removed = 0;
 217 uintx                    StringDedupTable::_resize_count = 0;
 218 uintx                    StringDedupTable::_rehash_count = 0;
 219 
 220 StringDedupTable*        StringDedupTable::_resized_table = NULL;
 221 StringDedupTable*        StringDedupTable::_rehashed_table = NULL;
 222 volatile size_t          StringDedupTable::_claimed_index = 0;
 223 
 224 StringDedupTable::StringDedupTable(size_t size, jint hash_seed) :
 225   _size(size),
 226   _entries(0),
 227   _grow_threshold((uintx)(size * _grow_load_factor)),
 228   _shrink_threshold((uintx)(size * _shrink_load_factor)),
 229   _rehash_needed(false),
 230   _hash_seed(hash_seed) {
 231   assert(is_power_of_2(size), "Table size must be a power of 2");
 232   _buckets = NEW_C_HEAP_ARRAY(StringDedupEntry*, _size, mtGC);
 233   memset(_buckets, 0, _size * sizeof(StringDedupEntry*));
 234 }
 235 
 236 StringDedupTable::~StringDedupTable() {
 237   FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets);
 238 }
 239 
 240 void StringDedupTable::create() {
 241   assert(_table == NULL, "One string deduplication table allowed");
 242   _entry_cache = new StringDedupEntryCache(_min_size * _max_cache_factor);
 243   _table = new StringDedupTable(_min_size);
 244 }
 245 
 246 void StringDedupTable::add(typeArrayOop value, bool latin1, unsigned int hash, StringDedupEntry** list) {
 247   StringDedupEntry* entry = _entry_cache->alloc();
 248   entry->set_obj(value);
 249   entry->set_hash(hash);
 250   entry->set_latin1(latin1);
 251   entry->set_next(*list);
 252   *list = entry;
 253   _entries++;
 254 }
 255 
 256 void StringDedupTable::remove(StringDedupEntry** pentry, uint worker_id) {
 257   StringDedupEntry* entry = *pentry;
 258   *pentry = entry->next();
 259   _entry_cache->free(entry, worker_id);
 260 }
 261 
 262 void StringDedupTable::transfer(StringDedupEntry** pentry, StringDedupTable* dest) {
 263   StringDedupEntry* entry = *pentry;
 264   *pentry = entry->next();
 265   unsigned int hash = entry->hash();
 266   size_t index = dest->hash_to_index(hash);
 267   StringDedupEntry** list = dest->bucket(index);
 268   entry->set_next(*list);
 269   *list = entry;
 270 }
 271 
 272 bool StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) {
 273   return (oopDesc::equals(value1, value2) ||
 274           (value1->length() == value2->length() &&
 275           (!memcmp(value1->base(T_BYTE),
 276                     value2->base(T_BYTE),
 277                     value1->length() * sizeof(jbyte)))));
 278 }
 279 
 280 typeArrayOop StringDedupTable::lookup(typeArrayOop value, bool latin1, unsigned int hash,
 281                                       StringDedupEntry** list, uintx &count) {
 282   for (StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) {
 283     if (entry->hash() == hash && entry->latin1() == latin1) {
 284       typeArrayOop existing_value = entry->obj();
 285       if (equals(value, existing_value)) {
 286         // Apply proper barrier to make sure it is kept alive. Concurrent mark might
 287         // otherwise declare it dead if there are no other strong references to this object.
 288         oop* obj_addr = (oop*)entry->obj_addr();
 289         oop obj = RootAccess<IN_CONCURRENT_ROOT>::oop_load(obj_addr);
 290         return typeArrayOop(obj);
 291       }
 292     }
 293     count++;
 294   }
 295 
 296   // Not found
 297   return NULL;
 298 }
 299 
 300 typeArrayOop StringDedupTable::lookup_or_add_inner(typeArrayOop value, bool latin1, unsigned int hash) {
 301   size_t index = hash_to_index(hash);
 302   StringDedupEntry** list = bucket(index);
 303   uintx count = 0;
 304 
 305   // Lookup in list
 306   typeArrayOop existing_value = lookup(value, latin1, hash, list, count);
 307 
 308   // Check if rehash is needed
 309   if (count > _rehash_threshold) {
 310     _rehash_needed = true;
 311   }
 312 
 313   if (existing_value == NULL) {
 314     // Not found, add new entry
 315     add(value, latin1, hash, list);
 316 
 317     // Update statistics
 318     _entries_added++;
 319   }
 320 
 321   return existing_value;
 322 }
 323 
 324 unsigned int StringDedupTable::hash_code(typeArrayOop value, bool latin1) {
 325   unsigned int hash;
 326   int length = value->length();
 327   if (latin1) {
 328     const jbyte* data = (jbyte*)value->base(T_BYTE);
 329     if (use_java_hash()) {
 330       hash = java_lang_String::hash_code(data, length);
 331     } else {
 332       hash = AltHashing::murmur3_32(_table->_hash_seed, data, length);
 333     }
 334   } else {
 335     length /= sizeof(jchar) / sizeof(jbyte); // Convert number of bytes to number of chars
 336     const jchar* data = (jchar*)value->base(T_CHAR);
 337     if (use_java_hash()) {
 338       hash = java_lang_String::hash_code(data, length);
 339     } else {
 340       hash = AltHashing::murmur3_32(_table->_hash_seed, data, length);
 341     }
 342   }
 343 
 344   return hash;
 345 }
 346 
 347 void StringDedupTable::deduplicate(oop java_string, StringDedupStat* stat) {
 348   assert(java_lang_String::is_instance(java_string), "Must be a string");
 349   NoSafepointVerifier nsv;
 350 
 351   stat->inc_inspected();
 352 
 353   typeArrayOop value = java_lang_String::value(java_string);
 354   if (value == NULL) {
 355     // String has no value
 356     stat->inc_skipped();
 357     return;
 358   }
 359 
 360   bool latin1 = java_lang_String::is_latin1(java_string);
 361   unsigned int hash = 0;
 362 
 363   if (use_java_hash()) {
 364     // Get hash code from cache
 365     hash = java_lang_String::hash(java_string);
 366   }
 367 
 368   if (hash == 0) {
 369     // Compute hash
 370     hash = hash_code(value, latin1);
 371     stat->inc_hashed();
 372 
 373     if (use_java_hash() && hash != 0) {
 374       // Store hash code in cache
 375       java_lang_String::set_hash(java_string, hash);
 376     }
 377   }
 378 
 379   typeArrayOop existing_value = lookup_or_add(value, latin1, hash);
 380   if (existing_value == value) {
 381     // Same value, already known
 382     stat->inc_known();
 383     return;
 384   }
 385 
 386   // Get size of value array
 387   uintx size_in_bytes = value->size() * HeapWordSize;
 388   stat->inc_new(size_in_bytes);
 389 
 390   if (existing_value != NULL) {




 391     // Existing value found, deduplicate string
 392     java_lang_String::set_value(java_string, existing_value);
 393     stat->deduped(value, size_in_bytes);





 394   }
 395 }
 396 
 397 bool StringDedupTable::is_resizing() {
 398   return _resized_table != NULL;
 399 }
 400 
 401 bool StringDedupTable::is_rehashing() {
 402   return _rehashed_table != NULL;
 403 }
 404 
 405 StringDedupTable* StringDedupTable::prepare_resize() {
 406   size_t size = _table->_size;
 407 
 408   // Check if the hashtable needs to be resized
 409   if (_table->_entries > _table->_grow_threshold) {
 410     // Grow table, double the size
 411     size *= 2;
 412     if (size > _max_size) {
 413       // Too big, don't resize
 414       return NULL;
 415     }
 416   } else if (_table->_entries < _table->_shrink_threshold) {
 417     // Shrink table, half the size
 418     size /= 2;
 419     if (size < _min_size) {
 420       // Too small, don't resize
 421       return NULL;
 422     }
 423   } else if (StringDeduplicationResizeALot) {
 424     // Force grow
 425     size *= 2;
 426     if (size > _max_size) {
 427       // Too big, force shrink instead
 428       size /= 4;
 429     }
 430   } else {
 431     // Resize not needed
 432     return NULL;
 433   }
 434 
 435   // Update statistics
 436   _resize_count++;
 437 
 438   // Update max cache size
 439   _entry_cache->set_max_size(size * _max_cache_factor);
 440 
 441   // Allocate the new table. The new table will be populated by workers
 442   // calling unlink_or_oops_do() and finally installed by finish_resize().
 443   return new StringDedupTable(size, _table->_hash_seed);
 444 }
 445 
 446 void StringDedupTable::finish_resize(StringDedupTable* resized_table) {
 447   assert(resized_table != NULL, "Invalid table");
 448 
 449   resized_table->_entries = _table->_entries;
 450 
 451   // Free old table
 452   delete _table;
 453 
 454   // Install new table
 455   _table = resized_table;
 456 }
 457 
 458 void StringDedupTable::unlink_or_oops_do(StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) {
 459   // The table is divided into partitions to allow lock-less parallel processing by
 460   // multiple worker threads. A worker thread first claims a partition, which ensures
 461   // exclusive access to that part of the table, then continues to process it. To allow
 462   // shrinking of the table in parallel we also need to make sure that the same worker
 463   // thread processes all partitions where entries will hash to the same destination
 464   // partition. Since the table size is always a power of two and we always shrink by
 465   // dividing the table in half, we know that for a given partition there is only one
 466   // other partition whoes entries will hash to the same destination partition. That
 467   // other partition is always the sibling partition in the second half of the table.
 468   // For example, if the table is divided into 8 partitions, the sibling of partition 0
 469   // is partition 4, the sibling of partition 1 is partition 5, etc.
 470   size_t table_half = _table->_size / 2;
 471 
 472   // Let each partition be one page worth of buckets
 473   size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(StringDedupEntry*));
 474   assert(table_half % partition_size == 0, "Invalid partition size");
 475 
 476   // Number of entries removed during the scan
 477   uintx removed = 0;
 478 
 479   for (;;) {
 480     // Grab next partition to scan
 481     size_t partition_begin = claim_table_partition(partition_size);
 482     size_t partition_end = partition_begin + partition_size;
 483     if (partition_begin >= table_half) {
 484       // End of table
 485       break;
 486     }
 487 
 488     // Scan the partition followed by the sibling partition in the second half of the table
 489     removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id);
 490     removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id);
 491   }
 492 
 493   // Delayed update to avoid contention on the table lock
 494   if (removed > 0) {
 495     MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag);
 496     _table->_entries -= removed;
 497     _entries_removed += removed;
 498   }
 499 }
 500 
 501 uintx StringDedupTable::unlink_or_oops_do(StringDedupUnlinkOrOopsDoClosure* cl,
 502                                           size_t partition_begin,
 503                                           size_t partition_end,
 504                                           uint worker_id) {
 505   uintx removed = 0;
 506   for (size_t bucket = partition_begin; bucket < partition_end; bucket++) {
 507     StringDedupEntry** entry = _table->bucket(bucket);
 508     while (*entry != NULL) {
 509       oop* p = (oop*)(*entry)->obj_addr();
 510       if (cl->is_alive(*p)) {
 511         cl->keep_alive(p);
 512         if (is_resizing()) {
 513           // We are resizing the table, transfer entry to the new table
 514           _table->transfer(entry, _resized_table);
 515         } else {
 516           if (is_rehashing()) {
 517             // We are rehashing the table, rehash the entry but keep it
 518             // in the table. We can't transfer entries into the new table
 519             // at this point since we don't have exclusive access to all
 520             // destination partitions. finish_rehash() will do a single
 521             // threaded transfer of all entries.
 522             typeArrayOop value = (typeArrayOop)*p;
 523             bool latin1 = (*entry)->latin1();
 524             unsigned int hash = hash_code(value, latin1);
 525             (*entry)->set_hash(hash);
 526           }
 527 
 528           // Move to next entry
 529           entry = (*entry)->next_addr();
 530         }
 531       } else {
 532         // Not alive, remove entry from table
 533         _table->remove(entry, worker_id);
 534         removed++;
 535       }
 536     }
 537   }
 538 
 539   return removed;
 540 }
 541 
 542 void StringDedupTable::gc_prologue(bool resize_and_rehash_table) {
 543   assert(!is_resizing() && !is_rehashing(), "Already in progress?");
 544 
 545   _claimed_index = 0;
 546   if (resize_and_rehash_table) {
 547     // If both resize and rehash is needed, only do resize. Rehash of
 548     // the table will eventually happen if the situation persists.
 549     _resized_table = StringDedupTable::prepare_resize();
 550     if (!is_resizing()) {
 551       _rehashed_table = StringDedupTable::prepare_rehash();
 552     }
 553   }
 554 }
 555 
 556 void StringDedupTable::gc_epilogue() {
 557   assert(!is_resizing() || !is_rehashing(), "Can not both resize and rehash");
 558   assert(_claimed_index >= _table->_size / 2 || _claimed_index == 0, "All or nothing");
 559 
 560   if (is_resizing()) {
 561     StringDedupTable::finish_resize(_resized_table);
 562     _resized_table = NULL;
 563   } else if (is_rehashing()) {
 564     StringDedupTable::finish_rehash(_rehashed_table);
 565     _rehashed_table = NULL;
 566   }
 567 }
 568 
 569 StringDedupTable* StringDedupTable::prepare_rehash() {
 570   if (!_table->_rehash_needed && !StringDeduplicationRehashALot) {
 571     // Rehash not needed
 572     return NULL;
 573   }
 574 
 575   // Update statistics
 576   _rehash_count++;
 577 
 578   // Compute new hash seed
 579   _table->_hash_seed = AltHashing::compute_seed();
 580 
 581   // Allocate the new table, same size and hash seed
 582   return new StringDedupTable(_table->_size, _table->_hash_seed);
 583 }
 584 
 585 void StringDedupTable::finish_rehash(StringDedupTable* rehashed_table) {
 586   assert(rehashed_table != NULL, "Invalid table");
 587 
 588   // Move all newly rehashed entries into the correct buckets in the new table
 589   for (size_t bucket = 0; bucket < _table->_size; bucket++) {
 590     StringDedupEntry** entry = _table->bucket(bucket);
 591     while (*entry != NULL) {
 592       _table->transfer(entry, rehashed_table);
 593     }
 594   }
 595 
 596   rehashed_table->_entries = _table->_entries;
 597 
 598   // Free old table
 599   delete _table;
 600 
 601   // Install new table
 602   _table = rehashed_table;
 603 }
 604 
 605 size_t StringDedupTable::claim_table_partition(size_t partition_size) {
 606   return Atomic::add(partition_size, &_claimed_index) - partition_size;
 607 }
 608 
 609 void StringDedupTable::verify() {
 610   for (size_t bucket = 0; bucket < _table->_size; bucket++) {
 611     // Verify entries
 612     StringDedupEntry** entry = _table->bucket(bucket);
 613     while (*entry != NULL) {
 614       typeArrayOop value = (*entry)->obj();
 615       guarantee(value != NULL, "Object must not be NULL");
 616       guarantee(Universe::heap()->is_in_reserved(value), "Object must be on the heap");
 617       guarantee(!value->is_forwarded(), "Object must not be forwarded");
 618       guarantee(value->is_typeArray(), "Object must be a typeArrayOop");
 619       bool latin1 = (*entry)->latin1();
 620       unsigned int hash = hash_code(value, latin1);
 621       guarantee((*entry)->hash() == hash, "Table entry has inorrect hash");
 622       guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index");
 623       entry = (*entry)->next_addr();
 624     }
 625 
 626     // Verify that we do not have entries with identical oops or identical arrays.
 627     // We only need to compare entries in the same bucket. If the same oop or an
 628     // identical array has been inserted more than once into different/incorrect
 629     // buckets the verification step above will catch that.
 630     StringDedupEntry** entry1 = _table->bucket(bucket);
 631     while (*entry1 != NULL) {
 632       typeArrayOop value1 = (*entry1)->obj();
 633       bool latin1_1 = (*entry1)->latin1();
 634       StringDedupEntry** entry2 = (*entry1)->next_addr();
 635       while (*entry2 != NULL) {
 636         typeArrayOop value2 = (*entry2)->obj();
 637         bool latin1_2 = (*entry2)->latin1();
 638         guarantee(latin1_1 != latin1_2 || !equals(value1, value2), "Table entries must not have identical arrays");
 639         entry2 = (*entry2)->next_addr();
 640       }
 641       entry1 = (*entry1)->next_addr();
 642     }
 643   }
 644 }
 645 
 646 void StringDedupTable::clean_entry_cache() {
 647   _entry_cache->delete_overflowed();
 648 }
 649 
 650 void StringDedupTable::print_statistics() {
 651   Log(gc, stringdedup) log;
 652   log.debug("  Table");
 653   log.debug("    Memory Usage: " STRDEDUP_BYTES_FORMAT_NS,
 654             STRDEDUP_BYTES_PARAM(_table->_size * sizeof(StringDedupEntry*) + (_table->_entries + _entry_cache->size()) * sizeof(StringDedupEntry)));
 655   log.debug("    Size: " SIZE_FORMAT ", Min: " SIZE_FORMAT ", Max: " SIZE_FORMAT, _table->_size, _min_size, _max_size);
 656   log.debug("    Entries: " UINTX_FORMAT ", Load: " STRDEDUP_PERCENT_FORMAT_NS ", Cached: " UINTX_FORMAT ", Added: " UINTX_FORMAT ", Removed: " UINTX_FORMAT,
 657             _table->_entries, percent_of(_table->_entries, _table->_size), _entry_cache->size(), _entries_added, _entries_removed);
 658   log.debug("    Resize Count: " UINTX_FORMAT ", Shrink Threshold: " UINTX_FORMAT "(" STRDEDUP_PERCENT_FORMAT_NS "), Grow Threshold: " UINTX_FORMAT "(" STRDEDUP_PERCENT_FORMAT_NS ")",
 659             _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0);
 660   log.debug("    Rehash Count: " UINTX_FORMAT ", Rehash Threshold: " UINTX_FORMAT ", Hash Seed: 0x%x", _rehash_count, _rehash_threshold, _table->_hash_seed);
 661   log.debug("    Age Threshold: " UINTX_FORMAT, StringDeduplicationAgeThreshold);
 662 }