1 /*
2 * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/altHashing.hpp"
27 #include "classfile/javaClasses.inline.hpp"
28 #include "gc/g1/g1CollectedHeap.inline.hpp"
29 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
30 #include "gc/g1/g1StringDedup.hpp"
31 #include "gc/g1/g1StringDedupTable.hpp"
32 #include "gc/shared/gcLocker.hpp"
33 #include "logging/log.hpp"
34 #include "memory/padded.inline.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "oops/typeArrayOop.hpp"
37 #include "runtime/mutexLocker.hpp"
38
39 //
40 // Freelist in the deduplication table entry cache. Links table
41 // entries together using their _next fields.
42 //
43 class G1StringDedupEntryFreeList : public CHeapObj<mtGC> {
44 private:
45 G1StringDedupEntry* _list;
46 size_t _length;
47
48 public:
49 G1StringDedupEntryFreeList() :
50 _list(NULL),
51 _length(0) {
52 }
53
54 void add(G1StringDedupEntry* entry) {
55 entry->set_next(_list);
56 _list = entry;
57 _length++;
58 }
59
60 G1StringDedupEntry* remove() {
61 G1StringDedupEntry* entry = _list;
62 if (entry != NULL) {
63 _list = entry->next();
64 _length--;
65 }
66 return entry;
67 }
68
69 size_t length() {
70 return _length;
71 }
72 };
73
74 //
75 // Cache of deduplication table entries. This cache provides fast allocation and
76 // reuse of table entries to lower the pressure on the underlying allocator.
77 // But more importantly, it provides fast/deferred freeing of table entries. This
78 // is important because freeing of table entries is done during stop-the-world
79 // phases and it is not uncommon for large number of entries to be freed at once.
80 // Tables entries that are freed during these phases are placed onto a freelist in
81 // the cache. The deduplication thread, which executes in a concurrent phase, will
82 // later reuse or free the underlying memory for these entries.
83 //
84 // The cache allows for single-threaded allocations and multi-threaded frees.
85 // Allocations are synchronized by StringDedupTable_lock as part of a table
86 // modification.
87 //
88 class G1StringDedupEntryCache : public CHeapObj<mtGC> {
89 private:
90 // One freelist per GC worker to allow lock less freeing of
91 // entries while doing a parallel scan of the table. Using
92 // PaddedEnd to avoid false sharing.
93 PaddedEnd<G1StringDedupEntryFreeList>* _lists;
94 size_t _nlists;
95
96 public:
97 G1StringDedupEntryCache();
98 ~G1StringDedupEntryCache();
99
100 // Get a table entry from the cache freelist, or allocate a new
101 // entry if the cache is empty.
102 G1StringDedupEntry* alloc();
103
104 // Insert a table entry into the cache freelist.
105 void free(G1StringDedupEntry* entry, uint worker_id);
106
107 // Returns current number of entries in the cache.
108 size_t size();
109
110 // If the cache has grown above the given max size, trim it down
111 // and deallocate the memory occupied by trimmed of entries.
112 void trim(size_t max_size);
113 };
114
115 G1StringDedupEntryCache::G1StringDedupEntryCache() {
116 _nlists = ParallelGCThreads;
117 _lists = PaddedArray<G1StringDedupEntryFreeList, mtGC>::create_unfreeable((uint)_nlists);
118 }
119
120 G1StringDedupEntryCache::~G1StringDedupEntryCache() {
121 ShouldNotReachHere();
122 }
123
124 G1StringDedupEntry* G1StringDedupEntryCache::alloc() {
125 for (size_t i = 0; i < _nlists; i++) {
126 G1StringDedupEntry* entry = _lists[i].remove();
127 if (entry != NULL) {
128 return entry;
129 }
130 }
131 return new G1StringDedupEntry();
132 }
133
134 void G1StringDedupEntryCache::free(G1StringDedupEntry* entry, uint worker_id) {
135 assert(entry->obj() != NULL, "Double free");
136 assert(worker_id < _nlists, "Invalid worker id");
137 entry->set_obj(NULL);
138 entry->set_hash(0);
139 _lists[worker_id].add(entry);
140 }
141
142 size_t G1StringDedupEntryCache::size() {
143 size_t size = 0;
144 for (size_t i = 0; i < _nlists; i++) {
145 size += _lists[i].length();
146 }
147 return size;
148 }
149
150 void G1StringDedupEntryCache::trim(size_t max_size) {
151 size_t cache_size = 0;
152 for (size_t i = 0; i < _nlists; i++) {
153 G1StringDedupEntryFreeList* list = &_lists[i];
154 cache_size += list->length();
155 while (cache_size > max_size) {
156 G1StringDedupEntry* entry = list->remove();
157 assert(entry != NULL, "Should not be null");
158 cache_size--;
159 delete entry;
160 }
161 }
162 }
163
164 G1StringDedupTable* G1StringDedupTable::_table = NULL;
165 G1StringDedupEntryCache* G1StringDedupTable::_entry_cache = NULL;
166
167 const size_t G1StringDedupTable::_min_size = (1 << 10); // 1024
168 const size_t G1StringDedupTable::_max_size = (1 << 24); // 16777216
169 const double G1StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load
170 const double G1StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load
171 const double G1StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size
172 const uintx G1StringDedupTable::_rehash_multiple = 60; // Hash bucket has 60 times more collisions than expected
173 const uintx G1StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor);
174
175 uintx G1StringDedupTable::_entries_added = 0;
176 uintx G1StringDedupTable::_entries_removed = 0;
177 uintx G1StringDedupTable::_resize_count = 0;
178 uintx G1StringDedupTable::_rehash_count = 0;
179
180 G1StringDedupTable::G1StringDedupTable(size_t size, jint hash_seed) :
181 _size(size),
182 _entries(0),
183 _grow_threshold((uintx)(size * _grow_load_factor)),
184 _shrink_threshold((uintx)(size * _shrink_load_factor)),
185 _rehash_needed(false),
186 _hash_seed(hash_seed) {
187 assert(is_power_of_2(size), "Table size must be a power of 2");
188 _buckets = NEW_C_HEAP_ARRAY(G1StringDedupEntry*, _size, mtGC);
189 memset(_buckets, 0, _size * sizeof(G1StringDedupEntry*));
190 }
191
192 G1StringDedupTable::~G1StringDedupTable() {
193 FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets);
194 }
195
196 void G1StringDedupTable::create() {
197 assert(_table == NULL, "One string deduplication table allowed");
198 _entry_cache = new G1StringDedupEntryCache();
199 _table = new G1StringDedupTable(_min_size);
200 }
201
202 void G1StringDedupTable::add(typeArrayOop value, bool latin1, unsigned int hash, G1StringDedupEntry** list) {
203 G1StringDedupEntry* entry = _entry_cache->alloc();
204 entry->set_obj(value);
205 entry->set_hash(hash);
206 entry->set_latin1(latin1);
207 entry->set_next(*list);
208 *list = entry;
209 _entries++;
210 }
211
212 void G1StringDedupTable::remove(G1StringDedupEntry** pentry, uint worker_id) {
213 G1StringDedupEntry* entry = *pentry;
214 *pentry = entry->next();
215 _entry_cache->free(entry, worker_id);
216 }
217
218 void G1StringDedupTable::transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest) {
219 G1StringDedupEntry* entry = *pentry;
220 *pentry = entry->next();
221 unsigned int hash = entry->hash();
222 size_t index = dest->hash_to_index(hash);
223 G1StringDedupEntry** list = dest->bucket(index);
224 entry->set_next(*list);
225 *list = entry;
226 }
227
228 bool G1StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) {
229 return (value1 == value2 ||
230 (value1->length() == value2->length() &&
231 (!memcmp(value1->base(T_BYTE),
232 value2->base(T_BYTE),
233 value1->length() * sizeof(jbyte)))));
234 }
235
236 typeArrayOop G1StringDedupTable::lookup(typeArrayOop value, bool latin1, unsigned int hash,
237 G1StringDedupEntry** list, uintx &count) {
238 for (G1StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) {
239 if (entry->hash() == hash && entry->latin1() == latin1) {
240 typeArrayOop existing_value = entry->obj();
241 if (equals(value, existing_value)) {
242 // Match found
243 return existing_value;
244 }
245 }
246 count++;
247 }
248
249 // Not found
250 return NULL;
251 }
252
253 typeArrayOop G1StringDedupTable::lookup_or_add_inner(typeArrayOop value, bool latin1, unsigned int hash) {
254 size_t index = hash_to_index(hash);
255 G1StringDedupEntry** list = bucket(index);
256 uintx count = 0;
257
258 // Lookup in list
259 typeArrayOop existing_value = lookup(value, latin1, hash, list, count);
260
261 // Check if rehash is needed
262 if (count > _rehash_threshold) {
263 _rehash_needed = true;
264 }
265
266 if (existing_value == NULL) {
267 // Not found, add new entry
268 add(value, latin1, hash, list);
269
270 // Update statistics
271 _entries_added++;
272 }
273
274 return existing_value;
275 }
276
277 unsigned int G1StringDedupTable::hash_code(typeArrayOop value, bool latin1) {
278 unsigned int hash;
279 int length = value->length();
280 if (latin1) {
281 const jbyte* data = (jbyte*)value->base(T_BYTE);
282 if (use_java_hash()) {
283 hash = java_lang_String::hash_code(data, length);
284 } else {
285 hash = AltHashing::murmur3_32(_table->_hash_seed, data, length);
286 }
287 } else {
288 length /= sizeof(jchar) / sizeof(jbyte); // Convert number of bytes to number of chars
289 const jchar* data = (jchar*)value->base(T_CHAR);
290 if (use_java_hash()) {
291 hash = java_lang_String::hash_code(data, length);
292 } else {
293 hash = AltHashing::murmur3_32(_table->_hash_seed, data, length);
294 }
295 }
296
297 return hash;
298 }
299
300 void G1StringDedupTable::deduplicate(oop java_string, G1StringDedupStat& stat) {
301 assert(java_lang_String::is_instance(java_string), "Must be a string");
302 No_Safepoint_Verifier nsv;
303
304 stat.inc_inspected();
305
306 typeArrayOop value = java_lang_String::value(java_string);
307 if (value == NULL) {
308 // String has no value
309 stat.inc_skipped();
310 return;
311 }
312
313 bool latin1 = java_lang_String::is_latin1(java_string);
314 unsigned int hash = 0;
315
316 if (use_java_hash()) {
317 // Get hash code from cache
318 hash = java_lang_String::hash(java_string);
319 }
320
321 if (hash == 0) {
322 // Compute hash
323 hash = hash_code(value, latin1);
324 stat.inc_hashed();
325
326 if (use_java_hash() && hash != 0) {
327 // Store hash code in cache
328 java_lang_String::set_hash(java_string, hash);
329 }
330 }
331
332 typeArrayOop existing_value = lookup_or_add(value, latin1, hash);
333 if (existing_value == value) {
334 // Same value, already known
335 stat.inc_known();
336 return;
337 }
338
339 // Get size of value array
340 uintx size_in_bytes = value->size() * HeapWordSize;
341 stat.inc_new(size_in_bytes);
342
343 if (existing_value != NULL) {
344 // Enqueue the reference to make sure it is kept alive. Concurrent mark might
345 // otherwise declare it dead if there are no other strong references to this object.
346 G1SATBCardTableModRefBS::enqueue(existing_value);
347
348 // Existing value found, deduplicate string
349 java_lang_String::set_value(java_string, existing_value);
350
351 if (G1CollectedHeap::heap()->is_in_young(value)) {
352 stat.inc_deduped_young(size_in_bytes);
353 } else {
354 stat.inc_deduped_old(size_in_bytes);
355 }
356 }
357 }
358
359 G1StringDedupTable* G1StringDedupTable::prepare_resize() {
360 size_t size = _table->_size;
361
362 // Check if the hashtable needs to be resized
363 if (_table->_entries > _table->_grow_threshold) {
364 // Grow table, double the size
365 size *= 2;
366 if (size > _max_size) {
367 // Too big, don't resize
368 return NULL;
369 }
370 } else if (_table->_entries < _table->_shrink_threshold) {
371 // Shrink table, half the size
372 size /= 2;
373 if (size < _min_size) {
374 // Too small, don't resize
375 return NULL;
376 }
377 } else if (StringDeduplicationResizeALot) {
378 // Force grow
379 size *= 2;
380 if (size > _max_size) {
381 // Too big, force shrink instead
382 size /= 4;
383 }
384 } else {
385 // Resize not needed
386 return NULL;
387 }
388
389 // Update statistics
390 _resize_count++;
391
392 // Allocate the new table. The new table will be populated by workers
393 // calling unlink_or_oops_do() and finally installed by finish_resize().
394 return new G1StringDedupTable(size, _table->_hash_seed);
395 }
396
397 void G1StringDedupTable::finish_resize(G1StringDedupTable* resized_table) {
398 assert(resized_table != NULL, "Invalid table");
399
400 resized_table->_entries = _table->_entries;
401
402 // Free old table
403 delete _table;
404
405 // Install new table
406 _table = resized_table;
407 }
408
409 void G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) {
410 // The table is divided into partitions to allow lock-less parallel processing by
411 // multiple worker threads. A worker thread first claims a partition, which ensures
412 // exclusive access to that part of the table, then continues to process it. To allow
413 // shrinking of the table in parallel we also need to make sure that the same worker
414 // thread processes all partitions where entries will hash to the same destination
415 // partition. Since the table size is always a power of two and we always shrink by
416 // dividing the table in half, we know that for a given partition there is only one
417 // other partition whoes entries will hash to the same destination partition. That
418 // other partition is always the sibling partition in the second half of the table.
419 // For example, if the table is divided into 8 partitions, the sibling of partition 0
420 // is partition 4, the sibling of partition 1 is partition 5, etc.
421 size_t table_half = _table->_size / 2;
422
423 // Let each partition be one page worth of buckets
424 size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(G1StringDedupEntry*));
425 assert(table_half % partition_size == 0, "Invalid partition size");
426
427 // Number of entries removed during the scan
428 uintx removed = 0;
429
430 for (;;) {
431 // Grab next partition to scan
432 size_t partition_begin = cl->claim_table_partition(partition_size);
433 size_t partition_end = partition_begin + partition_size;
434 if (partition_begin >= table_half) {
435 // End of table
436 break;
437 }
438
439 // Scan the partition followed by the sibling partition in the second half of the table
440 removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id);
441 removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id);
442 }
443
444 // Delayed update avoid contention on the table lock
445 if (removed > 0) {
446 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag);
447 _table->_entries -= removed;
448 _entries_removed += removed;
449 }
450 }
451
452 uintx G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl,
453 size_t partition_begin,
454 size_t partition_end,
455 uint worker_id) {
456 uintx removed = 0;
457 for (size_t bucket = partition_begin; bucket < partition_end; bucket++) {
458 G1StringDedupEntry** entry = _table->bucket(bucket);
459 while (*entry != NULL) {
460 oop* p = (oop*)(*entry)->obj_addr();
461 if (cl->is_alive(*p)) {
462 cl->keep_alive(p);
463 if (cl->is_resizing()) {
464 // We are resizing the table, transfer entry to the new table
465 _table->transfer(entry, cl->resized_table());
466 } else {
467 if (cl->is_rehashing()) {
468 // We are rehashing the table, rehash the entry but keep it
469 // in the table. We can't transfer entries into the new table
470 // at this point since we don't have exclusive access to all
471 // destination partitions. finish_rehash() will do a single
472 // threaded transfer of all entries.
473 typeArrayOop value = (typeArrayOop)*p;
474 bool latin1 = (*entry)->latin1();
475 unsigned int hash = hash_code(value, latin1);
476 (*entry)->set_hash(hash);
477 }
478
479 // Move to next entry
480 entry = (*entry)->next_addr();
481 }
482 } else {
483 // Not alive, remove entry from table
484 _table->remove(entry, worker_id);
485 removed++;
486 }
487 }
488 }
489
490 return removed;
491 }
492
493 G1StringDedupTable* G1StringDedupTable::prepare_rehash() {
494 if (!_table->_rehash_needed && !StringDeduplicationRehashALot) {
495 // Rehash not needed
496 return NULL;
497 }
498
499 // Update statistics
500 _rehash_count++;
501
502 // Compute new hash seed
503 _table->_hash_seed = AltHashing::compute_seed();
504
505 // Allocate the new table, same size and hash seed
506 return new G1StringDedupTable(_table->_size, _table->_hash_seed);
507 }
508
509 void G1StringDedupTable::finish_rehash(G1StringDedupTable* rehashed_table) {
510 assert(rehashed_table != NULL, "Invalid table");
511
512 // Move all newly rehashed entries into the correct buckets in the new table
513 for (size_t bucket = 0; bucket < _table->_size; bucket++) {
514 G1StringDedupEntry** entry = _table->bucket(bucket);
515 while (*entry != NULL) {
516 _table->transfer(entry, rehashed_table);
517 }
518 }
519
520 rehashed_table->_entries = _table->_entries;
521
522 // Free old table
523 delete _table;
524
525 // Install new table
526 _table = rehashed_table;
527 }
528
529 void G1StringDedupTable::verify() {
530 for (size_t bucket = 0; bucket < _table->_size; bucket++) {
531 // Verify entries
532 G1StringDedupEntry** entry = _table->bucket(bucket);
533 while (*entry != NULL) {
534 typeArrayOop value = (*entry)->obj();
535 guarantee(value != NULL, "Object must not be NULL");
536 guarantee(G1CollectedHeap::heap()->is_in_reserved(value), "Object must be on the heap");
537 guarantee(!value->is_forwarded(), "Object must not be forwarded");
538 guarantee(value->is_typeArray(), "Object must be a typeArrayOop");
539 bool latin1 = (*entry)->latin1();
540 unsigned int hash = hash_code(value, latin1);
541 guarantee((*entry)->hash() == hash, "Table entry has inorrect hash");
542 guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index");
543 entry = (*entry)->next_addr();
544 }
545
546 // Verify that we do not have entries with identical oops or identical arrays.
547 // We only need to compare entries in the same bucket. If the same oop or an
548 // identical array has been inserted more than once into different/incorrect
549 // buckets the verification step above will catch that.
550 G1StringDedupEntry** entry1 = _table->bucket(bucket);
551 while (*entry1 != NULL) {
552 typeArrayOop value1 = (*entry1)->obj();
553 bool latin1_1 = (*entry1)->latin1();
554 G1StringDedupEntry** entry2 = (*entry1)->next_addr();
555 while (*entry2 != NULL) {
556 typeArrayOop value2 = (*entry2)->obj();
557 bool latin1_2 = (*entry2)->latin1();
558 guarantee(latin1_1 != latin1_2 || !equals(value1, value2), "Table entries must not have identical arrays");
559 entry2 = (*entry2)->next_addr();
560 }
561 entry1 = (*entry1)->next_addr();
562 }
563 }
564 }
565
566 void G1StringDedupTable::trim_entry_cache() {
567 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag);
568 size_t max_cache_size = (size_t)(_table->_size * _max_cache_factor);
569 _entry_cache->trim(max_cache_size);
570 }
571
572 void G1StringDedupTable::print_statistics() {
573 LogHandle(gc, stringdedup) log;
574 log.debug(" [Table]");
575 log.debug(" [Memory Usage: " G1_STRDEDUP_BYTES_FORMAT_NS "]",
576 G1_STRDEDUP_BYTES_PARAM(_table->_size * sizeof(G1StringDedupEntry*) + (_table->_entries + _entry_cache->size()) * sizeof(G1StringDedupEntry)));
577 log.debug(" [Size: " SIZE_FORMAT ", Min: " SIZE_FORMAT ", Max: " SIZE_FORMAT "]", _table->_size, _min_size, _max_size);
578 log.debug(" [Entries: " UINTX_FORMAT ", Load: " G1_STRDEDUP_PERCENT_FORMAT_NS ", Cached: " UINTX_FORMAT ", Added: " UINTX_FORMAT ", Removed: " UINTX_FORMAT "]",
579 _table->_entries, (double)_table->_entries / (double)_table->_size * 100.0, _entry_cache->size(), _entries_added, _entries_removed);
580 log.debug(" [Resize Count: " UINTX_FORMAT ", Shrink Threshold: " UINTX_FORMAT "(" G1_STRDEDUP_PERCENT_FORMAT_NS "), Grow Threshold: " UINTX_FORMAT "(" G1_STRDEDUP_PERCENT_FORMAT_NS ")]",
581 _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0);
582 log.debug(" [Rehash Count: " UINTX_FORMAT ", Rehash Threshold: " UINTX_FORMAT ", Hash Seed: 0x%x]", _rehash_count, _rehash_threshold, _table->_hash_seed);
583 log.debug(" [Age Threshold: " UINTX_FORMAT "]", StringDeduplicationAgeThreshold);
584 }