1 /*
2 * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/altHashing.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
29 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
30 #include "gc_implementation/g1/g1StringDedupTable.hpp"
31 #include "memory/gcLocker.hpp"
32 #include "memory/padded.inline.hpp"
33 #include "oops/typeArrayOop.hpp"
34 #include "runtime/mutexLocker.hpp"
35
36 //
37 // Freelist in the deduplication table entry cache. Links table
38 // entries together using their _next fields.
39 //
40 class G1StringDedupEntryFreeList : public CHeapObj<mtGC> {
41 private:
42 G1StringDedupEntry* _list;
43 size_t _length;
44
45 public:
46 G1StringDedupEntryFreeList() :
47 _list(NULL),
48 _length(0) {
49 }
50
51 void add(G1StringDedupEntry* entry) {
52 entry->set_next(_list);
53 _list = entry;
54 _length++;
55 }
56
57 G1StringDedupEntry* remove() {
58 G1StringDedupEntry* entry = _list;
59 if (entry != NULL) {
60 _list = entry->next();
61 _length--;
62 }
63 return entry;
64 }
65
66 size_t length() {
67 return _length;
68 }
69 };
70
71 //
72 // Cache of deduplication table enties. This cache provides fast allocation and
73 // reuse of table entries to lower the preasure on the underlaying allocator.
74 // But more importantly, it provides fast/deferred freeing of table entries. This
75 // is important because freeing of table entries is done during stop-the-world
76 // phases and it is not uncommon for large number of String to be freed at once.
77 // Tables entries that are freed during these phases are placed onto a freelist in
78 // the cache. The deduplication thread, which executes in a concurrent phase, will
79 // later reuse or free the underlaying memory for these entries.
80 //
81 // The cache allows for single-threaded allocations and multi-threaded frees.
82 // Allocations are synchronized by StringDedupTable_lock as part of a table
83 // modification.
84 //
85 class G1StringDedupEntryCache : public CHeapObj<mtGC> {
86 private:
87 // One freelist per GC worker to allow lock less freeing of
88 // entries while doing a parallel scan of the table. Using
89 // PaddedEnd to avoid false sharing.
90 PaddedEnd<G1StringDedupEntryFreeList>* _lists;
91 size_t _nlists;
92
93 public:
94 G1StringDedupEntryCache();
95 ~G1StringDedupEntryCache();
96
97 // Get a table entry from the cache freelist, or allocate a new
98 // entry if the cache is empty.
99 G1StringDedupEntry* alloc();
100
101 // Insert a table entry into the cache freelist.
102 void free(G1StringDedupEntry* entry, uint worker_id);
103
104 // If the cache has grown above the given max size, trim it down
105 // and deallocate the memory occupied by trimmed of entries.
106 void trim(size_t max_size);
107 };
108
109 G1StringDedupEntryCache::G1StringDedupEntryCache() {
110 _nlists = MAX2(ParallelGCThreads, (size_t)1);
111 _lists = PaddedArray<G1StringDedupEntryFreeList, mtGC>::create_unfreeable((uint)_nlists);
112 }
113
114 G1StringDedupEntryCache::~G1StringDedupEntryCache() {
115 ShouldNotReachHere();
116 }
117
118 G1StringDedupEntry* G1StringDedupEntryCache::alloc() {
119 for (size_t i = 0; i < _nlists; i++) {
120 G1StringDedupEntry* entry = _lists[i].remove();
121 if (entry != NULL) {
122 return entry;
123 }
124 }
125 return new G1StringDedupEntry();
126 }
127
128 void G1StringDedupEntryCache::free(G1StringDedupEntry* entry, uint worker_id) {
129 assert(entry->obj() != NULL, "Double free");
130 assert(worker_id < _nlists, "Invalid worker id");
131 entry->set_obj(NULL);
132 entry->set_hash(0);
133 _lists[worker_id].add(entry);
134 }
135
136 void G1StringDedupEntryCache::trim(size_t max_size) {
137 size_t cache_size = 0;
138 for (size_t i = 0; i < _nlists; i++) {
139 G1StringDedupEntryFreeList* list = &_lists[i];
140 cache_size += list->length();
141 while (cache_size > max_size) {
142 G1StringDedupEntry* entry = list->remove();
143 assert(entry != NULL, "Should not be null");
144 cache_size--;
145 delete entry;
146 }
147 }
148 }
149
150 G1StringDedupTable* G1StringDedupTable::_table = NULL;
151 G1StringDedupEntryCache* G1StringDedupTable::_entry_cache = NULL;
152
153 const size_t G1StringDedupTable::_min_size = (1 << 10); // 1024
154 const size_t G1StringDedupTable::_max_size = (1 << 24); // 16777216
155 const double G1StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load
156 const double G1StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load
157 const double G1StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size
158 const uintx G1StringDedupTable::_rehash_multiple = 60; // Hash bucket has 60 times more collisions than expected
159 const uintx G1StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor);
160
161 uintx G1StringDedupTable::_entries_added = 0;
162 uintx G1StringDedupTable::_entries_removed = 0;
163 uintx G1StringDedupTable::_resize_count = 0;
164 uintx G1StringDedupTable::_rehash_count = 0;
165
166 G1StringDedupTable::G1StringDedupTable(size_t size, jint hash_seed) :
167 _size(size),
168 _entries(0),
169 _grow_threshold((uintx)(size * _grow_load_factor)),
170 _shrink_threshold((uintx)(size * _shrink_load_factor)),
171 _rehash_needed(false),
172 _hash_seed(hash_seed) {
173 assert(is_power_of_2(size), "Table size must be a power of 2");
174 _buckets = NEW_C_HEAP_ARRAY(G1StringDedupEntry*, _size, mtGC);
175 memset(_buckets, 0, _size * sizeof(G1StringDedupEntry*));
176 }
177
178 G1StringDedupTable::~G1StringDedupTable() {
179 FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets, mtGC);
180 }
181
182 void G1StringDedupTable::create() {
183 assert(_table == NULL, "One string deduplication table allowed");
184 _entry_cache = new G1StringDedupEntryCache();
185 _table = new G1StringDedupTable(_min_size);
186 }
187
188 void G1StringDedupTable::add(typeArrayOop value, unsigned int hash, G1StringDedupEntry** list) {
189 G1StringDedupEntry* entry = _entry_cache->alloc();
190 entry->set_obj(value);
191 entry->set_hash(hash);
192 entry->set_next(*list);
193 *list = entry;
194 _entries++;
195 }
196
197 void G1StringDedupTable::remove(G1StringDedupEntry** pentry, uint worker_id) {
198 G1StringDedupEntry* entry = *pentry;
199 *pentry = entry->next();
200 _entry_cache->free(entry, worker_id);
201 }
202
203 void G1StringDedupTable::transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest) {
204 G1StringDedupEntry* entry = *pentry;
205 *pentry = entry->next();
206 unsigned int hash = entry->hash();
207 size_t index = dest->hash_to_index(hash);
208 G1StringDedupEntry** list = dest->bucket(index);
209 entry->set_next(*list);
210 *list = entry;
211 }
212
213 bool G1StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) {
214 return (value1 == value2 ||
215 (value1->length() == value2->length() &&
216 (!memcmp(value1->base(T_CHAR),
217 value2->base(T_CHAR),
218 value1->length() * sizeof(jchar)))));
219 }
220
221 typeArrayOop G1StringDedupTable::lookup(typeArrayOop value, unsigned int hash,
222 G1StringDedupEntry** list, uintx &count) {
223 for (G1StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) {
224 if (entry->hash() == hash) {
225 typeArrayOop existing_value = entry->obj();
226 if (equals(value, existing_value)) {
227 // Match found
228 return existing_value;
229 }
230 }
231 count++;
232 }
233
234 // Not found
235 return NULL;
236 }
237
238 typeArrayOop G1StringDedupTable::lookup_or_add_inner(typeArrayOop value, unsigned int hash) {
239 size_t index = hash_to_index(hash);
240 G1StringDedupEntry** list = bucket(index);
241 uintx count = 0;
242
243 // Lookup in list
244 typeArrayOop existing_value = lookup(value, hash, list, count);
245
246 // Check if rehash is needed
247 if (count > _rehash_threshold) {
248 _rehash_needed = true;
249 }
250
251 if (existing_value == NULL) {
252 // Not found, add new entry
253 add(value, hash, list);
254
255 // Update statistics
256 _entries_added++;
257 }
258
259 return existing_value;
260 }
261
262 unsigned int G1StringDedupTable::hash_code(typeArrayOop value) {
263 unsigned int hash;
264 int length = value->length();
265 const jchar* data = (jchar*)value->base(T_CHAR);
266
267 if (use_java_hash()) {
268 hash = java_lang_String::hash_code(data, length);
269 } else {
270 hash = AltHashing::murmur3_32(_table->_hash_seed, data, length);
271 }
272
273 return hash;
274 }
275
276 void G1StringDedupTable::deduplicate(oop java_string, G1StringDedupStat& stat) {
277 assert(java_lang_String::is_instance(java_string), "Must be a string");
278 No_Safepoint_Verifier nsv;
279
280 stat.inc_inspected();
281
282 typeArrayOop value = java_lang_String::value(java_string);
283 if (value == NULL) {
284 // String has no value
285 stat.inc_skipped();
286 return;
287 }
288
289 unsigned int hash = 0;
290
291 if (use_java_hash()) {
292 // Get hash code from cache
293 hash = java_lang_String::hash(java_string);
294 }
295
296 if (hash == 0) {
297 // Compute hash
298 hash = hash_code(value);
299 stat.inc_hashed();
300 }
301
302 if (use_java_hash() && hash != 0) {
303 // Store hash code in cache
304 java_lang_String::set_hash(java_string, hash);
305 }
306
307 typeArrayOop existing_value = lookup_or_add(value, hash);
308 if (existing_value == value) {
309 // Same value, already known
310 stat.inc_known();
311 return;
312 }
313
314 // Get size of value array
315 uintx size_in_bytes = value->size() * HeapWordSize;
316 stat.inc_new(size_in_bytes);
317
318 if (existing_value != NULL) {
319 // Enqueue the reference to make sure it is kept alive. Concurrent mark might
320 // otherwise declare it dead if there are no other strong references to this object.
321 G1SATBCardTableModRefBS::enqueue(existing_value);
322
323 // Existing value found, deduplicate string
324 java_lang_String::set_value(java_string, existing_value);
325
326 if (G1CollectedHeap::heap()->is_in_young(value)) {
327 stat.inc_deduped_young(size_in_bytes);
328 } else {
329 stat.inc_deduped_old(size_in_bytes);
330 }
331 }
332 }
333
334 G1StringDedupTable* G1StringDedupTable::prepare_resize() {
335 size_t size = _table->_size;
336
337 // Check if the hashtable needs to be resized
338 if (_table->_entries > _table->_grow_threshold) {
339 // Grow table, double the size
340 size *= 2;
341 if (size > _max_size) {
342 // Too big, don't resize
343 return NULL;
344 }
345 } else if (_table->_entries < _table->_shrink_threshold) {
346 // Shrink table, half the size
347 size /= 2;
348 if (size < _min_size) {
349 // Too small, don't resize
350 return NULL;
351 }
352 } else if (StringDeduplicationResizeALot) {
353 // Force grow
354 size *= 2;
355 if (size > _max_size) {
356 // Too big, force shrink instead
357 size /= 4;
358 }
359 } else {
360 // Resize not needed
361 return NULL;
362 }
363
364 // Update statistics
365 _resize_count++;
366
367 // Allocate the new table. The new table will be populated by workers
368 // calling unlink_or_oops_do() and finally installed by finish_resize().
369 return new G1StringDedupTable(size, _table->_hash_seed);
370 }
371
372 void G1StringDedupTable::finish_resize(G1StringDedupTable* resized_table) {
373 assert(resized_table != NULL, "Invalid table");
374
375 resized_table->_entries = _table->_entries;
376
377 // Free old table
378 delete _table;
379
380 // Install new table
381 _table = resized_table;
382 }
383
384 void G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) {
385 // The table is divided into partitions to allow lock-less parallel processing by
386 // multiple worker threads. A worker thread first claims a partition, which ensures
387 // exclusive access to that part of the table, then continues to process it. To allow
388 // shrinking of the table in parallel we also need to make sure that the same worker
389 // thread processes all partitions where entries will hash to the same destination
390 // partition. Since the table size is always a power of two and we always shrink by
391 // dividing the table in half, we know that for a given partition there is only one
392 // other partition whoes entries will hash to the same destination partition. That
393 // other partition is always the sibling partition in the second half of the table.
394 // For example, if the table is divided into 8 partitions, the sibling of partition 0
395 // is partition 4, the sibling of partition 1 is partition 5, etc.
396 size_t table_half = _table->_size / 2;
397
398 // Let each partition be one page worth of buckets
399 size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(G1StringDedupEntry*));
400 assert(table_half % partition_size == 0, "Invalid partition size");
401
402 // Number of entries removed during the scan
403 uintx removed = 0;
404
405 for (;;) {
406 // Grab next partition to scan
407 size_t partition_begin = cl->claim_table_partition(partition_size);
408 size_t partition_end = partition_begin + partition_size;
409 if (partition_begin >= table_half) {
410 // End of table
411 break;
412 }
413
414 // Scan the partition followed by the sibling partition in the second half of the table
415 removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id);
416 removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id);
417 }
418
419 // Delayed update avoid contention on the table lock
420 if (removed > 0) {
421 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag);
422 _table->_entries -= removed;
423 _entries_removed += removed;
424 }
425 }
426
427 uintx G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl,
428 size_t partition_begin,
429 size_t partition_end,
430 uint worker_id) {
431 uintx removed = 0;
432 for (size_t bucket = partition_begin; bucket < partition_end; bucket++) {
433 G1StringDedupEntry** entry = _table->bucket(bucket);
434 while (*entry != NULL) {
435 oop* p = (oop*)(*entry)->obj_addr();
436 if (cl->is_alive(*p)) {
437 cl->keep_alive(p);
438 if (cl->is_resizing()) {
439 // We are resizing the table, transfer entry to the new table
440 _table->transfer(entry, cl->resized_table());
441 } else {
442 if (cl->is_rehashing()) {
443 // We are rehashing the table, rehash the entry but keep it
444 // in the table. We can't transfer entries into the new table
445 // at this point since we don't have exclusive access to all
446 // destination partitions. finish_rehash() will do a single
447 // threaded transfer of all entries.
448 typeArrayOop value = (typeArrayOop)*p;
449 unsigned int hash = hash_code(value);
450 (*entry)->set_hash(hash);
451 }
452
453 // Move to next entry
454 entry = (*entry)->next_addr();
455 }
456 } else {
457 // Not alive, remove entry from table
458 _table->remove(entry, worker_id);
459 removed++;
460 }
461 }
462 }
463
464 return removed;
465 }
466
467 G1StringDedupTable* G1StringDedupTable::prepare_rehash() {
468 if (!_table->_rehash_needed && !StringDeduplicationRehashALot) {
469 // Rehash not needed
470 return NULL;
471 }
472
473 // Update statistics
474 _rehash_count++;
475
476 // Compute new hash seed
477 _table->_hash_seed = AltHashing::compute_seed();
478
479 // Allocate the new table, same size and hash seed
480 return new G1StringDedupTable(_table->_size, _table->_hash_seed);
481 }
482
483 void G1StringDedupTable::finish_rehash(G1StringDedupTable* rehashed_table) {
484 assert(rehashed_table != NULL, "Invalid table");
485
486 // Move all newly rehashed entries into the correct buckets in the new table
487 for (size_t bucket = 0; bucket < _table->_size; bucket++) {
488 G1StringDedupEntry** entry = _table->bucket(bucket);
489 while (*entry != NULL) {
490 _table->transfer(entry, rehashed_table);
491 }
492 }
493
494 rehashed_table->_entries = _table->_entries;
495
496 // Free old table
497 delete _table;
498
499 // Install new table
500 _table = rehashed_table;
501 }
502
503 void G1StringDedupTable::verify() {
504 for (size_t bucket = 0; bucket < _table->_size; bucket++) {
505 // Verify entries
506 G1StringDedupEntry** entry = _table->bucket(bucket);
507 while (*entry != NULL) {
508 typeArrayOop value = (*entry)->obj();
509 guarantee(value != NULL, "Object must not be NULL");
510 guarantee(Universe::heap()->is_in_reserved(value), "Object must be on the heap");
511 guarantee(!value->is_forwarded(), "Object must not be forwarded");
512 guarantee(value->is_typeArray(), "Object must be a typeArrayOop");
513 unsigned int hash = hash_code(value);
514 guarantee((*entry)->hash() == hash, "Table entry has inorrect hash");
515 guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index");
516 entry = (*entry)->next_addr();
517 }
518
519 // Verify that we do not have entries with identical oops or identical arrays.
520 // We only need to compare entries in the same bucket. If the same oop or an
521 // identical array has been inserted more than once into different/incorrect
522 // buckets the verification step above will catch that.
523 G1StringDedupEntry** entry1 = _table->bucket(bucket);
524 while (*entry1 != NULL) {
525 typeArrayOop value1 = (*entry1)->obj();
526 G1StringDedupEntry** entry2 = (*entry1)->next_addr();
527 while (*entry2 != NULL) {
528 typeArrayOop value2 = (*entry2)->obj();
529 guarantee(!equals(value1, value2), "Table entries must not have identical arrays");
530 entry2 = (*entry2)->next_addr();
531 }
532 entry1 = (*entry1)->next_addr();
533 }
534 }
535 }
536
537 void G1StringDedupTable::trim_entry_cache() {
538 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag);
539 size_t max_cache_size = (size_t)(_table->_size * _max_cache_factor);
540 _entry_cache->trim(max_cache_size);
541 }
542
543 void G1StringDedupTable::print_statistics(outputStream* st) {
544 st->print_cr(
545 " [Table]\n"
546 " [Memory Usage: "G1_STRDEDUP_BYTES_FORMAT_NS"]\n"
547 " [Size: "SIZE_FORMAT", Min: "SIZE_FORMAT", Max: "SIZE_FORMAT"]\n"
548 " [Entries: "UINTX_FORMAT", Load: "G1_STRDEDUP_PERCENT_FORMAT_NS", Added: "UINTX_FORMAT", Removed: "UINTX_FORMAT"]\n"
549 " [Resize Count: "UINTX_FORMAT", Shrink Threshold: "UINTX_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT_NS"), Grow Threshold: "UINTX_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT_NS")]\n"
550 " [Rehash Count: "UINTX_FORMAT", Rehash Threshold: "UINTX_FORMAT", Hash Seed: 0x%x]\n"
551 " [Age Threshold: "UINTX_FORMAT"]",
552 G1_STRDEDUP_BYTES_PARAM(_table->_size * sizeof(G1StringDedupEntry*) + _table->_entries * sizeof(G1StringDedupEntry)),
553 _table->_size, _min_size, _max_size,
554 _table->_entries, (double)_table->_entries / (double)_table->_size * 100.0, _entries_added, _entries_removed,
555 _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0,
556 _rehash_count, _rehash_threshold, _table->_hash_seed,
557 StringDeduplicationAgeThreshold);
558 }