1 /*
2 * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "memory/allocation.hpp"
27 #include "memory/universe.hpp"
28
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shared/gcTimer.hpp"
31 #include "gc/shared/gcTraceTime.inline.hpp"
32 #include "gc/shared/locationPrinter.inline.hpp"
33 #include "gc/shared/memAllocator.hpp"
34 #include "gc/shared/oopStorageSet.hpp"
35 #include "gc/shared/plab.hpp"
36
37 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
38 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
39 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
40 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
41 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
42 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
43 #include "gc/shenandoah/shenandoahControlThread.hpp"
44 #include "gc/shenandoah/shenandoahFreeSet.hpp"
45 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
46 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
47 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
48 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
49 #include "gc/shenandoah/shenandoahIUMode.hpp"
50 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
51 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
52 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
53 #include "gc/shenandoah/shenandoahMetrics.hpp"
54 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
55 #include "gc/shenandoah/shenandoahNormalMode.hpp"
56 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
57 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
58 #include "gc/shenandoah/shenandoahPadding.hpp"
59 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
60 #include "gc/shenandoah/shenandoahPassiveMode.hpp"
61 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
62 #include "gc/shenandoah/shenandoahStringDedup.hpp"
63 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
64 #include "gc/shenandoah/shenandoahUtils.hpp"
65 #include "gc/shenandoah/shenandoahVerifier.hpp"
66 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
67 #include "gc/shenandoah/shenandoahVMOperations.hpp"
68 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
69 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
70 #if INCLUDE_JFR
71 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
72 #endif
73
74 #include "memory/metaspace.hpp"
75 #include "oops/compressedOops.inline.hpp"
76 #include "runtime/atomic.hpp"
77 #include "runtime/globals.hpp"
78 #include "runtime/interfaceSupport.inline.hpp"
79 #include "runtime/orderAccess.hpp"
80 #include "runtime/safepointMechanism.hpp"
81 #include "runtime/vmThread.hpp"
82 #include "services/mallocTracker.hpp"
83 #include "utilities/powerOfTwo.hpp"
84
85 ShenandoahHeap* ShenandoahHeap::_heap = NULL;
86
87 #ifdef ASSERT
88 template <class T>
89 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
90 T o = RawAccess<>::oop_load(p);
91 if (! CompressedOops::is_null(o)) {
92 oop obj = CompressedOops::decode_not_null(o);
93 shenandoah_assert_not_forwarded(p, obj);
94 }
95 }
96
97 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
98 void ShenandoahAssertToSpaceClosure::do_oop(oop* p) { do_oop_work(p); }
99 #endif
100
101 class ShenandoahPretouchHeapTask : public AbstractGangTask {
102 private:
103 ShenandoahRegionIterator _regions;
104 const size_t _page_size;
105 public:
106 ShenandoahPretouchHeapTask(size_t page_size) :
107 AbstractGangTask("Shenandoah Pretouch Heap"),
108 _page_size(page_size) {}
109
110 virtual void work(uint worker_id) {
111 ShenandoahHeapRegion* r = _regions.next();
112 while (r != NULL) {
113 os::pretouch_memory(r->bottom(), r->end(), _page_size);
114 r = _regions.next();
115 }
116 }
117 };
118
119 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
120 private:
121 ShenandoahRegionIterator _regions;
122 char* _bitmap_base;
123 const size_t _bitmap_size;
124 const size_t _page_size;
125 public:
126 ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
127 AbstractGangTask("Shenandoah Pretouch Bitmap"),
128 _bitmap_base(bitmap_base),
129 _bitmap_size(bitmap_size),
130 _page_size(page_size) {}
131
132 virtual void work(uint worker_id) {
133 ShenandoahHeapRegion* r = _regions.next();
134 while (r != NULL) {
135 size_t start = r->index() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
136 size_t end = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
137 assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
138
139 os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
140
141 r = _regions.next();
142 }
143 }
144 };
145
146 jint ShenandoahHeap::initialize() {
147 initialize_heuristics();
148
149 //
150 // Figure out heap sizing
151 //
152
153 size_t init_byte_size = InitialHeapSize;
154 size_t min_byte_size = MinHeapSize;
155 size_t max_byte_size = MaxHeapSize;
156 size_t heap_alignment = HeapAlignment;
157
158 size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
159
160 if (ShenandoahAlwaysPreTouch) {
161 // Enabled pre-touch means the entire heap is committed right away.
162 init_byte_size = max_byte_size;
163 }
164
165 Universe::check_alignment(max_byte_size, reg_size_bytes, "Shenandoah heap");
166 Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
167
168 _num_regions = ShenandoahHeapRegion::region_count();
169
170 size_t num_committed_regions = init_byte_size / reg_size_bytes;
171 num_committed_regions = MIN2(num_committed_regions, _num_regions);
172 assert(num_committed_regions <= _num_regions, "sanity");
173 _initial_size = num_committed_regions * reg_size_bytes;
174
175 size_t num_min_regions = min_byte_size / reg_size_bytes;
176 num_min_regions = MIN2(num_min_regions, _num_regions);
177 assert(num_min_regions <= _num_regions, "sanity");
178 _minimum_size = num_min_regions * reg_size_bytes;
179
180 _committed = _initial_size;
181
182 size_t heap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
183 size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
184 size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
185
186 //
187 // Reserve and commit memory for heap
188 //
189
190 ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
191 initialize_reserved_region(heap_rs);
192 _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
193 _heap_region_special = heap_rs.special();
194
195 assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
196 "Misaligned heap: " PTR_FORMAT, p2i(base()));
197
198 #if SHENANDOAH_OPTIMIZED_OBJTASK
199 // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
200 // Fail if we ever attempt to address more than we can.
201 if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
202 FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
203 "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
204 "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
205 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
206 vm_exit_during_initialization("Fatal Error", buf);
207 }
208 #endif
209
210 ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
211 if (!_heap_region_special) {
212 os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
213 "Cannot commit heap memory");
214 }
215
216 //
217 // Reserve and commit memory for bitmap(s)
218 //
219
220 _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
221 _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
222
223 size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
224
225 guarantee(bitmap_bytes_per_region != 0,
226 "Bitmap bytes per region should not be zero");
227 guarantee(is_power_of_2(bitmap_bytes_per_region),
228 "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
229
230 if (bitmap_page_size > bitmap_bytes_per_region) {
231 _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
232 _bitmap_bytes_per_slice = bitmap_page_size;
233 } else {
234 _bitmap_regions_per_slice = 1;
235 _bitmap_bytes_per_slice = bitmap_bytes_per_region;
236 }
237
238 guarantee(_bitmap_regions_per_slice >= 1,
239 "Should have at least one region per slice: " SIZE_FORMAT,
240 _bitmap_regions_per_slice);
241
242 guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
243 "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
244 _bitmap_bytes_per_slice, bitmap_page_size);
245
246 ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
247 MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
248 _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
249 _bitmap_region_special = bitmap.special();
250
251 size_t bitmap_init_commit = _bitmap_bytes_per_slice *
252 align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
253 bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
254 if (!_bitmap_region_special) {
255 os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
256 "Cannot commit bitmap memory");
257 }
258
259 _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
260
261 if (ShenandoahVerify) {
262 ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
263 if (!verify_bitmap.special()) {
264 os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
265 "Cannot commit verification bitmap memory");
266 }
267 MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
268 MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
269 _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
270 _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
271 }
272
273 // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
274 ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
275 MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
276 _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
277 _aux_bitmap_region_special = aux_bitmap.special();
278 _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
279
280 //
281 // Create regions and region sets
282 //
283 size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
284 size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
285
286 ReservedSpace region_storage(region_storage_size, region_page_size);
287 MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
288 if (!region_storage.special()) {
289 os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
290 "Cannot commit region memory");
291 }
292
293 _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
294 _free_set = new ShenandoahFreeSet(this, _num_regions);
295 _collection_set = new ShenandoahCollectionSet(this, sh_rs.base(), sh_rs.size());
296
297 {
298 ShenandoahHeapLocker locker(lock());
299
300 for (size_t i = 0; i < _num_regions; i++) {
301 HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
302 bool is_committed = i < num_committed_regions;
303 void* loc = region_storage.base() + i * region_align;
304
305 ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
306 assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
307
308 _marking_context->initialize_top_at_mark_start(r);
309 _regions[i] = r;
310 assert(!collection_set()->is_in(i), "New region should not be in collection set");
311 }
312
313 // Initialize to complete
314 _marking_context->mark_complete();
315
316 _free_set->rebuild();
317 }
318
319 if (ShenandoahAlwaysPreTouch) {
320 assert(!AlwaysPreTouch, "Should have been overridden");
321
322 // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
323 // before initialize() below zeroes it with initializing thread. For any given region,
324 // we touch the region and the corresponding bitmaps from the same thread.
325 ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
326
327 size_t pretouch_heap_page_size = heap_page_size;
328 size_t pretouch_bitmap_page_size = bitmap_page_size;
329
330 #ifdef LINUX
331 // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
332 // pages. But, the kernel needs to know that every small page is used, in order to coalesce
333 // them into huge one. Therefore, we need to pretouch with smaller pages.
334 if (UseTransparentHugePages) {
335 pretouch_heap_page_size = (size_t)os::vm_page_size();
336 pretouch_bitmap_page_size = (size_t)os::vm_page_size();
337 }
338 #endif
339
340 // OS memory managers may want to coalesce back-to-back pages. Make their jobs
341 // simpler by pre-touching continuous spaces (heap and bitmap) separately.
342
343 log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
344 _num_regions, pretouch_bitmap_page_size);
345 ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size);
346 _workers->run_task(&bcl);
347
348 log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
349 _num_regions, pretouch_heap_page_size);
350 ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size);
351 _workers->run_task(&hcl);
352 }
353
354 //
355 // Initialize the rest of GC subsystems
356 //
357
358 _liveness_cache = NEW_C_HEAP_ARRAY(jushort*, _max_workers, mtGC);
359 for (uint worker = 0; worker < _max_workers; worker++) {
360 _liveness_cache[worker] = NEW_C_HEAP_ARRAY(jushort, _num_regions, mtGC);
361 Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(jushort));
362 }
363
364 // There should probably be Shenandoah-specific options for these,
365 // just as there are G1-specific options.
366 {
367 ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
368 satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
369 satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
370 }
371
372 _monitoring_support = new ShenandoahMonitoringSupport(this);
373 _phase_timings = new ShenandoahPhaseTimings();
374 ShenandoahStringDedup::initialize();
375 ShenandoahCodeRoots::initialize();
376
377 if (ShenandoahPacing) {
378 _pacer = new ShenandoahPacer(this);
379 _pacer->setup_for_idle();
380 } else {
381 _pacer = NULL;
382 }
383
384 _control_thread = new ShenandoahControlThread();
385
386 log_info(gc, init)("Initialize Shenandoah heap: " SIZE_FORMAT "%s initial, " SIZE_FORMAT "%s min, " SIZE_FORMAT "%s max",
387 byte_size_in_proper_unit(_initial_size), proper_unit_for_byte_size(_initial_size),
388 byte_size_in_proper_unit(_minimum_size), proper_unit_for_byte_size(_minimum_size),
389 byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity())
390 );
391
392 log_info(gc, init)("Safepointing mechanism: thread-local poll");
393
394 return JNI_OK;
395 }
396
397 void ShenandoahHeap::initialize_heuristics() {
398 if (ShenandoahGCMode != NULL) {
399 if (strcmp(ShenandoahGCMode, "normal") == 0) {
400 _gc_mode = new ShenandoahNormalMode();
401 } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
402 _gc_mode = new ShenandoahIUMode();
403 } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
404 _gc_mode = new ShenandoahPassiveMode();
405 } else {
406 vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
407 }
408 } else {
409 ShouldNotReachHere();
410 }
411 _gc_mode->initialize_flags();
412 if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
413 vm_exit_during_initialization(
414 err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
415 _gc_mode->name()));
416 }
417 if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
418 vm_exit_during_initialization(
419 err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
420 _gc_mode->name()));
421 }
422 log_info(gc, init)("Shenandoah GC mode: %s",
423 _gc_mode->name());
424
425 _heuristics = _gc_mode->initialize_heuristics();
426
427 if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
428 vm_exit_during_initialization(
429 err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
430 _heuristics->name()));
431 }
432 if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
433 vm_exit_during_initialization(
434 err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
435 _heuristics->name()));
436 }
437 log_info(gc, init)("Shenandoah heuristics: %s",
438 _heuristics->name());
439 }
440
441 #ifdef _MSC_VER
442 #pragma warning( push )
443 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
444 #endif
445
446 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
447 CollectedHeap(),
448 _initial_size(0),
449 _used(0),
450 _committed(0),
451 _bytes_allocated_since_gc_start(0),
452 _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
453 _workers(NULL),
454 _safepoint_workers(NULL),
455 _heap_region_special(false),
456 _num_regions(0),
457 _regions(NULL),
458 _update_refs_iterator(this),
459 _control_thread(NULL),
460 _shenandoah_policy(policy),
461 _heuristics(NULL),
462 _free_set(NULL),
463 _scm(new ShenandoahConcurrentMark()),
464 _full_gc(new ShenandoahMarkCompact()),
465 _pacer(NULL),
466 _verifier(NULL),
467 _phase_timings(NULL),
468 _monitoring_support(NULL),
469 _memory_pool(NULL),
470 _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
471 _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
472 _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
473 _soft_ref_policy(),
474 _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
475 _ref_processor(NULL),
476 _marking_context(NULL),
477 _bitmap_size(0),
478 _bitmap_regions_per_slice(0),
479 _bitmap_bytes_per_slice(0),
480 _bitmap_region_special(false),
481 _aux_bitmap_region_special(false),
482 _liveness_cache(NULL),
483 _collection_set(NULL)
484 {
485 _heap = this;
486
487 log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
488 log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial");
489
490 BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
491
492 _max_workers = MAX2(_max_workers, 1U);
493 _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
494 /* are_GC_task_threads */ true,
495 /* are_ConcurrentGC_threads */ true);
496 if (_workers == NULL) {
497 vm_exit_during_initialization("Failed necessary allocation.");
498 } else {
499 _workers->initialize_workers();
500 }
501
502 if (ParallelGCThreads > 1) {
503 _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
504 ParallelGCThreads,
505 /* are_GC_task_threads */ false,
506 /* are_ConcurrentGC_threads */ false);
507 _safepoint_workers->initialize_workers();
508 }
509 }
510
511 #ifdef _MSC_VER
512 #pragma warning( pop )
513 #endif
514
515 class ShenandoahResetBitmapTask : public AbstractGangTask {
516 private:
517 ShenandoahRegionIterator _regions;
518
519 public:
520 ShenandoahResetBitmapTask() :
521 AbstractGangTask("Parallel Reset Bitmap Task") {}
522
523 void work(uint worker_id) {
524 ShenandoahHeapRegion* region = _regions.next();
525 ShenandoahHeap* heap = ShenandoahHeap::heap();
526 ShenandoahMarkingContext* const ctx = heap->marking_context();
527 while (region != NULL) {
528 if (heap->is_bitmap_slice_committed(region)) {
529 ctx->clear_bitmap(region);
530 }
531 region = _regions.next();
532 }
533 }
534 };
535
536 void ShenandoahHeap::reset_mark_bitmap() {
537 assert_gc_workers(_workers->active_workers());
538 mark_incomplete_marking_context();
539
540 ShenandoahResetBitmapTask task;
541 _workers->run_task(&task);
542 }
543
544 void ShenandoahHeap::print_on(outputStream* st) const {
545 st->print_cr("Shenandoah Heap");
546 st->print_cr(" " SIZE_FORMAT "%s total, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
547 byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
548 byte_size_in_proper_unit(committed()), proper_unit_for_byte_size(committed()),
549 byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
550 st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
551 num_regions(),
552 byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
553 proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
554
555 st->print("Status: ");
556 if (has_forwarded_objects()) st->print("has forwarded objects, ");
557 if (is_concurrent_mark_in_progress()) st->print("marking, ");
558 if (is_evacuation_in_progress()) st->print("evacuating, ");
559 if (is_update_refs_in_progress()) st->print("updating refs, ");
560 if (is_degenerated_gc_in_progress()) st->print("degenerated gc, ");
561 if (is_full_gc_in_progress()) st->print("full gc, ");
562 if (is_full_gc_move_in_progress()) st->print("full gc move, ");
563 if (is_concurrent_root_in_progress()) st->print("concurrent roots, ");
564
565 if (cancelled_gc()) {
566 st->print("cancelled");
567 } else {
568 st->print("not cancelled");
569 }
570 st->cr();
571
572 st->print_cr("Reserved region:");
573 st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
574 p2i(reserved_region().start()),
575 p2i(reserved_region().end()));
576
577 ShenandoahCollectionSet* cset = collection_set();
578 st->print_cr("Collection set:");
579 if (cset != NULL) {
580 st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
581 st->print_cr(" - map (biased): " PTR_FORMAT, p2i(cset->biased_map_address()));
582 } else {
583 st->print_cr(" (NULL)");
584 }
585
586 st->cr();
587 MetaspaceUtils::print_on(st);
588
589 if (Verbose) {
590 print_heap_regions_on(st);
591 }
592 }
593
594 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
595 public:
596 void do_thread(Thread* thread) {
597 assert(thread != NULL, "Sanity");
598 assert(thread->is_Worker_thread(), "Only worker thread expected");
599 ShenandoahThreadLocalData::initialize_gclab(thread);
600 }
601 };
602
603 void ShenandoahHeap::post_initialize() {
604 CollectedHeap::post_initialize();
605 MutexLocker ml(Threads_lock);
606
607 ShenandoahInitWorkerGCLABClosure init_gclabs;
608 _workers->threads_do(&init_gclabs);
609
610 // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
611 // Now, we will let WorkGang to initialize gclab when new worker is created.
612 _workers->set_initialize_gclab();
613
614 _scm->initialize(_max_workers);
615 _full_gc->initialize(_gc_timer);
616
617 ref_processing_init();
618
619 _heuristics->initialize();
620
621 JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
622 }
623
624 size_t ShenandoahHeap::used() const {
625 return Atomic::load_acquire(&_used);
626 }
627
628 size_t ShenandoahHeap::committed() const {
629 OrderAccess::acquire();
630 return _committed;
631 }
632
633 void ShenandoahHeap::increase_committed(size_t bytes) {
634 shenandoah_assert_heaplocked_or_safepoint();
635 _committed += bytes;
636 }
637
638 void ShenandoahHeap::decrease_committed(size_t bytes) {
639 shenandoah_assert_heaplocked_or_safepoint();
640 _committed -= bytes;
641 }
642
643 void ShenandoahHeap::increase_used(size_t bytes) {
644 Atomic::add(&_used, bytes);
645 }
646
647 void ShenandoahHeap::set_used(size_t bytes) {
648 Atomic::release_store_fence(&_used, bytes);
649 }
650
651 void ShenandoahHeap::decrease_used(size_t bytes) {
652 assert(used() >= bytes, "never decrease heap size by more than we've left");
653 Atomic::sub(&_used, bytes);
654 }
655
656 void ShenandoahHeap::increase_allocated(size_t bytes) {
657 Atomic::add(&_bytes_allocated_since_gc_start, bytes);
658 }
659
660 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
661 size_t bytes = words * HeapWordSize;
662 if (!waste) {
663 increase_used(bytes);
664 }
665 increase_allocated(bytes);
666 if (ShenandoahPacing) {
667 control_thread()->pacing_notify_alloc(words);
668 if (waste) {
669 pacer()->claim_for_alloc(words, true);
670 }
671 }
672 }
673
674 size_t ShenandoahHeap::capacity() const {
675 return committed();
676 }
677
678 size_t ShenandoahHeap::max_capacity() const {
679 return _num_regions * ShenandoahHeapRegion::region_size_bytes();
680 }
681
682 size_t ShenandoahHeap::min_capacity() const {
683 return _minimum_size;
684 }
685
686 size_t ShenandoahHeap::initial_capacity() const {
687 return _initial_size;
688 }
689
690 bool ShenandoahHeap::is_in(const void* p) const {
691 HeapWord* heap_base = (HeapWord*) base();
692 HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
693 return p >= heap_base && p < last_region_end;
694 }
695
696 void ShenandoahHeap::op_uncommit(double shrink_before) {
697 assert (ShenandoahUncommit, "should be enabled");
698
699 // Application allocates from the beginning of the heap, and GC allocates at
700 // the end of it. It is more efficient to uncommit from the end, so that applications
701 // could enjoy the near committed regions. GC allocations are much less frequent,
702 // and therefore can accept the committing costs.
703
704 size_t count = 0;
705 for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
706 ShenandoahHeapRegion* r = get_region(i - 1);
707 if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
708 ShenandoahHeapLocker locker(lock());
709 if (r->is_empty_committed()) {
710 // Do not uncommit below minimal capacity
711 if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
712 break;
713 }
714
715 r->make_uncommitted();
716 count++;
717 }
718 }
719 SpinPause(); // allow allocators to take the lock
720 }
721
722 if (count > 0) {
723 control_thread()->notify_heap_changed();
724 }
725 }
726
727 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
728 // New object should fit the GCLAB size
729 size_t min_size = MAX2(size, PLAB::min_size());
730
731 // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
732 size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
733 new_size = MIN2(new_size, PLAB::max_size());
734 new_size = MAX2(new_size, PLAB::min_size());
735
736 // Record new heuristic value even if we take any shortcut. This captures
737 // the case when moderately-sized objects always take a shortcut. At some point,
738 // heuristics should catch up with them.
739 ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
740
741 if (new_size < size) {
742 // New size still does not fit the object. Fall back to shared allocation.
743 // This avoids retiring perfectly good GCLABs, when we encounter a large object.
744 return NULL;
745 }
746
747 // Retire current GCLAB, and allocate a new one.
748 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
749 gclab->retire();
750
751 size_t actual_size = 0;
752 HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
753 if (gclab_buf == NULL) {
754 return NULL;
755 }
756
757 assert (size <= actual_size, "allocation should fit");
758
759 if (ZeroTLAB) {
760 // ..and clear it.
761 Copy::zero_to_words(gclab_buf, actual_size);
762 } else {
763 // ...and zap just allocated object.
764 #ifdef ASSERT
765 // Skip mangling the space corresponding to the object header to
766 // ensure that the returned space is not considered parsable by
767 // any concurrent GC thread.
768 size_t hdr_size = oopDesc::header_size();
769 Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
770 #endif // ASSERT
771 }
772 gclab->set_buf(gclab_buf, actual_size);
773 return gclab->allocate(size);
774 }
775
776 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
777 size_t requested_size,
778 size_t* actual_size) {
779 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
780 HeapWord* res = allocate_memory(req);
781 if (res != NULL) {
782 *actual_size = req.actual_size();
783 } else {
784 *actual_size = 0;
785 }
786 return res;
787 }
788
789 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
790 size_t word_size,
791 size_t* actual_size) {
792 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
793 HeapWord* res = allocate_memory(req);
794 if (res != NULL) {
795 *actual_size = req.actual_size();
796 } else {
797 *actual_size = 0;
798 }
799 return res;
800 }
801
802 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
803 intptr_t pacer_epoch = 0;
804 bool in_new_region = false;
805 HeapWord* result = NULL;
806
807 if (req.is_mutator_alloc()) {
808 if (ShenandoahPacing) {
809 pacer()->pace_for_alloc(req.size());
810 pacer_epoch = pacer()->epoch();
811 }
812
813 if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
814 result = allocate_memory_under_lock(req, in_new_region);
815 }
816
817 // Allocation failed, block until control thread reacted, then retry allocation.
818 //
819 // It might happen that one of the threads requesting allocation would unblock
820 // way later after GC happened, only to fail the second allocation, because
821 // other threads have already depleted the free storage. In this case, a better
822 // strategy is to try again, as long as GC makes progress.
823 //
824 // Then, we need to make sure the allocation was retried after at least one
825 // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
826
827 size_t tries = 0;
828
829 while (result == NULL && _progress_last_gc.is_set()) {
830 tries++;
831 control_thread()->handle_alloc_failure(req);
832 result = allocate_memory_under_lock(req, in_new_region);
833 }
834
835 while (result == NULL && tries <= ShenandoahFullGCThreshold) {
836 tries++;
837 control_thread()->handle_alloc_failure(req);
838 result = allocate_memory_under_lock(req, in_new_region);
839 }
840
841 } else {
842 assert(req.is_gc_alloc(), "Can only accept GC allocs here");
843 result = allocate_memory_under_lock(req, in_new_region);
844 // Do not call handle_alloc_failure() here, because we cannot block.
845 // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
846 }
847
848 if (in_new_region) {
849 control_thread()->notify_heap_changed();
850 }
851
852 if (result != NULL) {
853 size_t requested = req.size();
854 size_t actual = req.actual_size();
855
856 assert (req.is_lab_alloc() || (requested == actual),
857 "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
858 ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
859
860 if (req.is_mutator_alloc()) {
861 notify_mutator_alloc_words(actual, false);
862
863 // If we requested more than we were granted, give the rest back to pacer.
864 // This only matters if we are in the same pacing epoch: do not try to unpace
865 // over the budget for the other phase.
866 if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
867 pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
868 }
869 } else {
870 increase_used(actual*HeapWordSize);
871 }
872 }
873
874 return result;
875 }
876
877 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
878 ShenandoahHeapLocker locker(lock());
879 return _free_set->allocate(req, in_new_region);
880 }
881
882 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
883 bool* gc_overhead_limit_was_exceeded) {
884 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
885 return allocate_memory(req);
886 }
887
888 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
889 size_t size,
890 Metaspace::MetadataType mdtype) {
891 MetaWord* result;
892
893 // Inform metaspace OOM to GC heuristics if class unloading is possible.
894 if (heuristics()->can_unload_classes()) {
895 ShenandoahHeuristics* h = heuristics();
896 h->record_metaspace_oom();
897 }
898
899 // Expand and retry allocation
900 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
901 if (result != NULL) {
902 return result;
903 }
904
905 // Start full GC
906 collect(GCCause::_metadata_GC_clear_soft_refs);
907
908 // Retry allocation
909 result = loader_data->metaspace_non_null()->allocate(size, mdtype);
910 if (result != NULL) {
911 return result;
912 }
913
914 // Expand and retry allocation
915 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
916 if (result != NULL) {
917 return result;
918 }
919
920 // Out of memory
921 return NULL;
922 }
923
924 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
925 private:
926 ShenandoahHeap* const _heap;
927 Thread* const _thread;
928 public:
929 ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
930 _heap(heap), _thread(Thread::current()) {}
931
932 void do_object(oop p) {
933 shenandoah_assert_marked(NULL, p);
934 if (!p->is_forwarded()) {
935 _heap->evacuate_object(p, _thread);
936 }
937 }
938 };
939
940 class ShenandoahEvacuationTask : public AbstractGangTask {
941 private:
942 ShenandoahHeap* const _sh;
943 ShenandoahCollectionSet* const _cs;
944 bool _concurrent;
945 public:
946 ShenandoahEvacuationTask(ShenandoahHeap* sh,
947 ShenandoahCollectionSet* cs,
948 bool concurrent) :
949 AbstractGangTask("Parallel Evacuation Task"),
950 _sh(sh),
951 _cs(cs),
952 _concurrent(concurrent)
953 {}
954
955 void work(uint worker_id) {
956 if (_concurrent) {
957 ShenandoahConcurrentWorkerSession worker_session(worker_id);
958 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
959 ShenandoahEvacOOMScope oom_evac_scope;
960 do_work();
961 } else {
962 ShenandoahParallelWorkerSession worker_session(worker_id);
963 ShenandoahEvacOOMScope oom_evac_scope;
964 do_work();
965 }
966 }
967
968 private:
969 void do_work() {
970 ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
971 ShenandoahHeapRegion* r;
972 while ((r =_cs->claim_next()) != NULL) {
973 assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
974 _sh->marked_object_iterate(r, &cl);
975
976 if (ShenandoahPacing) {
977 _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
978 }
979
980 if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
981 break;
982 }
983 }
984 }
985 };
986
987 void ShenandoahHeap::trash_cset_regions() {
988 ShenandoahHeapLocker locker(lock());
989
990 ShenandoahCollectionSet* set = collection_set();
991 ShenandoahHeapRegion* r;
992 set->clear_current_index();
993 while ((r = set->next()) != NULL) {
994 r->make_trash();
995 }
996 collection_set()->clear();
997 }
998
999 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1000 st->print_cr("Heap Regions:");
1001 st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1002 st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1003 st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark");
1004 st->print_cr("SN=alloc sequence number");
1005
1006 for (size_t i = 0; i < num_regions(); i++) {
1007 get_region(i)->print_on(st);
1008 }
1009 }
1010
1011 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1012 assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1013
1014 oop humongous_obj = oop(start->bottom());
1015 size_t size = humongous_obj->size();
1016 size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1017 size_t index = start->index() + required_regions - 1;
1018
1019 assert(!start->has_live(), "liveness must be zero");
1020
1021 for(size_t i = 0; i < required_regions; i++) {
1022 // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1023 // as it expects that every region belongs to a humongous region starting with a humongous start region.
1024 ShenandoahHeapRegion* region = get_region(index --);
1025
1026 assert(region->is_humongous(), "expect correct humongous start or continuation");
1027 assert(!region->is_cset(), "Humongous region should not be in collection set");
1028
1029 region->make_trash_immediate();
1030 }
1031 }
1032
1033 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1034 public:
1035 void do_thread(Thread* thread) {
1036 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1037 assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1038 gclab->retire();
1039 }
1040 };
1041
1042 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1043 if (UseTLAB) {
1044 CollectedHeap::ensure_parsability(retire_tlabs);
1045 }
1046 ShenandoahRetireGCLABClosure cl;
1047 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1048 cl.do_thread(t);
1049 }
1050 workers()->threads_do(&cl);
1051 }
1052
1053 void ShenandoahHeap::resize_tlabs() {
1054 CollectedHeap::resize_all_tlabs();
1055 }
1056
1057 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1058 private:
1059 ShenandoahRootEvacuator* _rp;
1060
1061 public:
1062 ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1063 AbstractGangTask("Shenandoah evacuate and update roots"),
1064 _rp(rp) {}
1065
1066 void work(uint worker_id) {
1067 ShenandoahParallelWorkerSession worker_session(worker_id);
1068 ShenandoahEvacOOMScope oom_evac_scope;
1069 ShenandoahEvacuateUpdateRootsClosure<> cl;
1070 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1071 _rp->roots_do(worker_id, &cl);
1072 }
1073 };
1074
1075 void ShenandoahHeap::evacuate_and_update_roots() {
1076 #if COMPILER2_OR_JVMCI
1077 DerivedPointerTable::clear();
1078 #endif
1079 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1080 {
1081 // Include concurrent roots if current cycle can not process those roots concurrently
1082 ShenandoahRootEvacuator rp(workers()->active_workers(),
1083 ShenandoahPhaseTimings::init_evac,
1084 !ShenandoahConcurrentRoots::should_do_concurrent_roots(),
1085 !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1086 ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1087 workers()->run_task(&roots_task);
1088 }
1089
1090 #if COMPILER2_OR_JVMCI
1091 DerivedPointerTable::update_pointers();
1092 #endif
1093 }
1094
1095 // Returns size in bytes
1096 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1097 if (ShenandoahElasticTLAB) {
1098 // With Elastic TLABs, return the max allowed size, and let the allocation path
1099 // figure out the safe size for current allocation.
1100 return ShenandoahHeapRegion::max_tlab_size_bytes();
1101 } else {
1102 return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1103 }
1104 }
1105
1106 size_t ShenandoahHeap::max_tlab_size() const {
1107 // Returns size in words
1108 return ShenandoahHeapRegion::max_tlab_size_words();
1109 }
1110
1111 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1112 public:
1113 void do_thread(Thread* thread) {
1114 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1115 gclab->retire();
1116 if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1117 ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1118 }
1119 }
1120 };
1121
1122 void ShenandoahHeap::retire_and_reset_gclabs() {
1123 ShenandoahRetireAndResetGCLABClosure cl;
1124 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1125 cl.do_thread(t);
1126 }
1127 workers()->threads_do(&cl);
1128 }
1129
1130 void ShenandoahHeap::collect(GCCause::Cause cause) {
1131 control_thread()->request_gc(cause);
1132 }
1133
1134 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1135 //assert(false, "Shouldn't need to do full collections");
1136 }
1137
1138 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1139 ShenandoahHeapRegion* r = heap_region_containing(addr);
1140 if (r != NULL) {
1141 return r->block_start(addr);
1142 }
1143 return NULL;
1144 }
1145
1146 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1147 ShenandoahHeapRegion* r = heap_region_containing(addr);
1148 return r->block_is_obj(addr);
1149 }
1150
1151 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1152 return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1153 }
1154
1155 jlong ShenandoahHeap::millis_since_last_gc() {
1156 double v = heuristics()->time_since_last_gc() * 1000;
1157 assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1158 return (jlong)v;
1159 }
1160
1161 void ShenandoahHeap::prepare_for_verify() {
1162 if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1163 make_parsable(false);
1164 }
1165 }
1166
1167 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1168 workers()->print_worker_threads_on(st);
1169 if (ShenandoahStringDedup::is_enabled()) {
1170 ShenandoahStringDedup::print_worker_threads_on(st);
1171 }
1172 }
1173
1174 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1175 workers()->threads_do(tcl);
1176 if (_safepoint_workers != NULL) {
1177 _safepoint_workers->threads_do(tcl);
1178 }
1179 if (ShenandoahStringDedup::is_enabled()) {
1180 ShenandoahStringDedup::threads_do(tcl);
1181 }
1182 }
1183
1184 void ShenandoahHeap::print_tracing_info() const {
1185 LogTarget(Info, gc, stats) lt;
1186 if (lt.is_enabled()) {
1187 ResourceMark rm;
1188 LogStream ls(lt);
1189
1190 phase_timings()->print_on(&ls);
1191
1192 ls.cr();
1193 ls.cr();
1194
1195 shenandoah_policy()->print_gc_stats(&ls);
1196
1197 ls.cr();
1198 ls.cr();
1199
1200 if (ShenandoahPacing) {
1201 pacer()->print_on(&ls);
1202 }
1203
1204 ls.cr();
1205 ls.cr();
1206 }
1207 }
1208
1209 void ShenandoahHeap::verify(VerifyOption vo) {
1210 if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1211 if (ShenandoahVerify) {
1212 verifier()->verify_generic(vo);
1213 } else {
1214 // TODO: Consider allocating verification bitmaps on demand,
1215 // and turn this on unconditionally.
1216 }
1217 }
1218 }
1219 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1220 return _free_set->capacity();
1221 }
1222
1223 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1224 private:
1225 MarkBitMap* _bitmap;
1226 Stack<oop,mtGC>* _oop_stack;
1227 ShenandoahHeap* const _heap;
1228 ShenandoahMarkingContext* const _marking_context;
1229
1230 template <class T>
1231 void do_oop_work(T* p) {
1232 T o = RawAccess<>::oop_load(p);
1233 if (!CompressedOops::is_null(o)) {
1234 oop obj = CompressedOops::decode_not_null(o);
1235 if (_heap->is_concurrent_root_in_progress() && !_marking_context->is_marked(obj)) {
1236 // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1237 return;
1238 }
1239 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1240
1241 assert(oopDesc::is_oop(obj), "must be a valid oop");
1242 if (!_bitmap->is_marked(obj)) {
1243 _bitmap->mark(obj);
1244 _oop_stack->push(obj);
1245 }
1246 }
1247 }
1248 public:
1249 ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1250 _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1251 _marking_context(_heap->marking_context()) {}
1252 void do_oop(oop* p) { do_oop_work(p); }
1253 void do_oop(narrowOop* p) { do_oop_work(p); }
1254 };
1255
1256 /*
1257 * This is public API, used in preparation of object_iterate().
1258 * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1259 * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1260 * control, we call SH::make_tlabs_parsable().
1261 */
1262 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1263 // No-op.
1264 }
1265
1266 /*
1267 * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1268 *
1269 * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1270 * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1271 * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1272 * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1273 * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1274 * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1275 * wiped the bitmap in preparation for next marking).
1276 *
1277 * For all those reasons, we implement object iteration as a single marking traversal, reporting
1278 * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1279 * is allowed to report dead objects, but is not required to do so.
1280 */
1281 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1282 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1283 if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1284 log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1285 return;
1286 }
1287
1288 // Reset bitmap
1289 _aux_bit_map.clear();
1290
1291 Stack<oop,mtGC> oop_stack;
1292
1293 // First, we process GC roots according to current GC cycle. This populates the work stack with initial objects.
1294 ShenandoahHeapIterationRootScanner rp;
1295 ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1296
1297 rp.roots_do(&oops);
1298
1299 // Work through the oop stack to traverse heap.
1300 while (! oop_stack.is_empty()) {
1301 oop obj = oop_stack.pop();
1302 assert(oopDesc::is_oop(obj), "must be a valid oop");
1303 cl->do_object(obj);
1304 obj->oop_iterate(&oops);
1305 }
1306
1307 assert(oop_stack.is_empty(), "should be empty");
1308
1309 if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1310 log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1311 }
1312 }
1313
1314 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1315 void ShenandoahHeap::keep_alive(oop obj) {
1316 if (is_concurrent_mark_in_progress()) {
1317 ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1318 }
1319 }
1320
1321 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1322 for (size_t i = 0; i < num_regions(); i++) {
1323 ShenandoahHeapRegion* current = get_region(i);
1324 blk->heap_region_do(current);
1325 }
1326 }
1327
1328 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1329 private:
1330 ShenandoahHeap* const _heap;
1331 ShenandoahHeapRegionClosure* const _blk;
1332
1333 shenandoah_padding(0);
1334 volatile size_t _index;
1335 shenandoah_padding(1);
1336
1337 public:
1338 ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1339 AbstractGangTask("Parallel Region Task"),
1340 _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1341
1342 void work(uint worker_id) {
1343 size_t stride = ShenandoahParallelRegionStride;
1344
1345 size_t max = _heap->num_regions();
1346 while (_index < max) {
1347 size_t cur = Atomic::fetch_and_add(&_index, stride);
1348 size_t start = cur;
1349 size_t end = MIN2(cur + stride, max);
1350 if (start >= max) break;
1351
1352 for (size_t i = cur; i < end; i++) {
1353 ShenandoahHeapRegion* current = _heap->get_region(i);
1354 _blk->heap_region_do(current);
1355 }
1356 }
1357 }
1358 };
1359
1360 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1361 assert(blk->is_thread_safe(), "Only thread-safe closures here");
1362 if (num_regions() > ShenandoahParallelRegionStride) {
1363 ShenandoahParallelHeapRegionTask task(blk);
1364 workers()->run_task(&task);
1365 } else {
1366 heap_region_iterate(blk);
1367 }
1368 }
1369
1370 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1371 private:
1372 ShenandoahMarkingContext* const _ctx;
1373 public:
1374 ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1375
1376 void heap_region_do(ShenandoahHeapRegion* r) {
1377 if (r->is_active()) {
1378 r->clear_live_data();
1379 _ctx->capture_top_at_mark_start(r);
1380 } else {
1381 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1382 assert(_ctx->top_at_mark_start(r) == r->top(),
1383 "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1384 }
1385 }
1386
1387 bool is_thread_safe() { return true; }
1388 };
1389
1390 void ShenandoahHeap::op_init_mark() {
1391 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1392 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1393
1394 assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1395 assert(!marking_context()->is_complete(), "should not be complete");
1396 assert(!has_forwarded_objects(), "No forwarded objects on this path");
1397
1398 if (ShenandoahVerify) {
1399 verifier()->verify_before_concmark();
1400 }
1401
1402 if (VerifyBeforeGC) {
1403 Universe::verify();
1404 }
1405
1406 set_concurrent_mark_in_progress(true);
1407 // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1408 {
1409 ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
1410 make_parsable(true);
1411 }
1412
1413 {
1414 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
1415 ShenandoahInitMarkUpdateRegionStateClosure cl;
1416 parallel_heap_region_iterate(&cl);
1417 }
1418
1419 // Make above changes visible to worker threads
1420 OrderAccess::fence();
1421
1422 concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1423
1424 if (UseTLAB) {
1425 ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1426 resize_tlabs();
1427 }
1428
1429 if (ShenandoahPacing) {
1430 pacer()->setup_for_mark();
1431 }
1432
1433 // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
1434 // we need to make sure that all its metadata are marked. alternative is to remark
1435 // thread roots at final mark pause, but it can be potential latency killer.
1436 if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1437 ShenandoahCodeRoots::arm_nmethods();
1438 }
1439 }
1440
1441 void ShenandoahHeap::op_mark() {
1442 concurrent_mark()->mark_from_roots();
1443 }
1444
1445 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1446 private:
1447 ShenandoahMarkingContext* const _ctx;
1448 ShenandoahHeapLock* const _lock;
1449
1450 public:
1451 ShenandoahFinalMarkUpdateRegionStateClosure() :
1452 _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1453
1454 void heap_region_do(ShenandoahHeapRegion* r) {
1455 if (r->is_active()) {
1456 // All allocations past TAMS are implicitly live, adjust the region data.
1457 // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1458 HeapWord *tams = _ctx->top_at_mark_start(r);
1459 HeapWord *top = r->top();
1460 if (top > tams) {
1461 r->increase_live_data_alloc_words(pointer_delta(top, tams));
1462 }
1463
1464 // We are about to select the collection set, make sure it knows about
1465 // current pinning status. Also, this allows trashing more regions that
1466 // now have their pinning status dropped.
1467 if (r->is_pinned()) {
1468 if (r->pin_count() == 0) {
1469 ShenandoahHeapLocker locker(_lock);
1470 r->make_unpinned();
1471 }
1472 } else {
1473 if (r->pin_count() > 0) {
1474 ShenandoahHeapLocker locker(_lock);
1475 r->make_pinned();
1476 }
1477 }
1478
1479 // Remember limit for updating refs. It's guaranteed that we get no
1480 // from-space-refs written from here on.
1481 r->set_update_watermark(r->top());
1482 } else {
1483 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1484 assert(_ctx->top_at_mark_start(r) == r->top(),
1485 "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1486 }
1487 }
1488
1489 bool is_thread_safe() { return true; }
1490 };
1491
1492 void ShenandoahHeap::op_final_mark() {
1493 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1494 assert(!has_forwarded_objects(), "No forwarded objects on this path");
1495
1496 // It is critical that we
1497 // evacuate roots right after finishing marking, so that we don't
1498 // get unmarked objects in the roots.
1499
1500 if (!cancelled_gc()) {
1501 concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1502
1503 // Marking is completed, deactivate SATB barrier
1504 set_concurrent_mark_in_progress(false);
1505 mark_complete_marking_context();
1506
1507 parallel_cleaning(false /* full gc*/);
1508
1509 if (ShenandoahVerify) {
1510 verifier()->verify_roots_no_forwarded();
1511 }
1512
1513 {
1514 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_region_states);
1515 ShenandoahFinalMarkUpdateRegionStateClosure cl;
1516 parallel_heap_region_iterate(&cl);
1517
1518 assert_pinned_region_status();
1519 }
1520
1521 // Force the threads to reacquire their TLABs outside the collection set.
1522 {
1523 ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
1524 make_parsable(true);
1525 }
1526
1527 {
1528 ShenandoahGCPhase phase(ShenandoahPhaseTimings::choose_cset);
1529 ShenandoahHeapLocker locker(lock());
1530 _collection_set->clear();
1531 heuristics()->choose_collection_set(_collection_set);
1532 }
1533
1534 {
1535 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_rebuild_freeset);
1536 ShenandoahHeapLocker locker(lock());
1537 _free_set->rebuild();
1538 }
1539
1540 if (!is_degenerated_gc_in_progress()) {
1541 prepare_concurrent_roots();
1542 prepare_concurrent_unloading();
1543 }
1544
1545 // If collection set has candidates, start evacuation.
1546 // Otherwise, bypass the rest of the cycle.
1547 if (!collection_set()->is_empty()) {
1548 ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1549
1550 if (ShenandoahVerify) {
1551 verifier()->verify_before_evacuation();
1552 }
1553
1554 set_evacuation_in_progress(true);
1555 // From here on, we need to update references.
1556 set_has_forwarded_objects(true);
1557
1558 if (!is_degenerated_gc_in_progress()) {
1559 if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1560 ShenandoahCodeRoots::arm_nmethods();
1561 }
1562 evacuate_and_update_roots();
1563 }
1564
1565 if (ShenandoahPacing) {
1566 pacer()->setup_for_evac();
1567 }
1568
1569 if (ShenandoahVerify) {
1570 ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None;
1571 if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1572 types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
1573 types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots);
1574 types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::StringDedupRoots);
1575 }
1576
1577 if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1578 types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots);
1579 }
1580 verifier()->verify_roots_no_forwarded_except(types);
1581 verifier()->verify_during_evacuation();
1582 }
1583 } else {
1584 if (ShenandoahVerify) {
1585 verifier()->verify_after_concmark();
1586 }
1587
1588 if (VerifyAfterGC) {
1589 Universe::verify();
1590 }
1591 }
1592
1593 } else {
1594 // If this cycle was updating references, we need to keep the has_forwarded_objects
1595 // flag on, for subsequent phases to deal with it.
1596 concurrent_mark()->cancel();
1597 set_concurrent_mark_in_progress(false);
1598
1599 if (process_references()) {
1600 // Abandon reference processing right away: pre-cleaning must have failed.
1601 ReferenceProcessor *rp = ref_processor();
1602 rp->disable_discovery();
1603 rp->abandon_partial_discovery();
1604 rp->verify_no_references_recorded();
1605 }
1606 }
1607 }
1608
1609 void ShenandoahHeap::op_conc_evac() {
1610 ShenandoahEvacuationTask task(this, _collection_set, true);
1611 workers()->run_task(&task);
1612 }
1613
1614 void ShenandoahHeap::op_stw_evac() {
1615 ShenandoahEvacuationTask task(this, _collection_set, false);
1616 workers()->run_task(&task);
1617 }
1618
1619 void ShenandoahHeap::op_updaterefs() {
1620 update_heap_references(true);
1621 }
1622
1623 void ShenandoahHeap::op_cleanup() {
1624 free_set()->recycle_trash();
1625 }
1626
1627 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1628 private:
1629 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1630 ShenandoahWeakRoots<true /*concurrent*/> _weak_roots;
1631 ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
1632 ShenandoahConcurrentStringDedupRoots _dedup_roots;
1633 bool _include_weak_roots;
1634
1635 public:
1636 ShenandoahConcurrentRootsEvacUpdateTask(bool include_weak_roots) :
1637 AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task"),
1638 _include_weak_roots(include_weak_roots) {
1639 }
1640
1641 void work(uint worker_id) {
1642 ShenandoahEvacOOMScope oom;
1643 {
1644 // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1645 // may race against OopStorage::release() calls.
1646 ShenandoahEvacUpdateOopStorageRootsClosure cl;
1647 _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl);
1648
1649 if (_include_weak_roots) {
1650 _weak_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl);
1651 }
1652 }
1653
1654 {
1655 ShenandoahEvacuateUpdateRootsClosure<> cl;
1656 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1657 _cld_roots.cld_do(&clds);
1658 }
1659
1660 {
1661 ShenandoahForwardedIsAliveClosure is_alive;
1662 ShenandoahEvacuateUpdateRootsClosure<MO_RELEASE> keep_alive;
1663 _dedup_roots.oops_do(&is_alive, &keep_alive, worker_id);
1664 }
1665 }
1666 };
1667
1668 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
1669 private:
1670 ShenandoahHeap* const _heap;
1671 ShenandoahMarkingContext* const _mark_context;
1672 bool _evac_in_progress;
1673 Thread* const _thread;
1674 size_t _dead_counter;
1675
1676 public:
1677 ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
1678 void do_oop(oop* p);
1679 void do_oop(narrowOop* p);
1680
1681 size_t dead_counter() const;
1682 void reset_dead_counter();
1683 };
1684
1685 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
1686 _heap(ShenandoahHeap::heap()),
1687 _mark_context(ShenandoahHeap::heap()->marking_context()),
1688 _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
1689 _thread(Thread::current()),
1690 _dead_counter(0) {
1691 }
1692
1693 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
1694 const oop obj = RawAccess<>::oop_load(p);
1695 if (!CompressedOops::is_null(obj)) {
1696 if (!_mark_context->is_marked(obj)) {
1697 shenandoah_assert_correct(p, obj);
1698 oop old = Atomic::cmpxchg(p, obj, oop(NULL));
1699 if (obj == old) {
1700 _dead_counter ++;
1701 }
1702 } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
1703 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1704 if (resolved == obj) {
1705 resolved = _heap->evacuate_object(obj, _thread);
1706 }
1707 Atomic::cmpxchg(p, obj, resolved);
1708 assert(_heap->cancelled_gc() ||
1709 _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
1710 "Sanity");
1711 }
1712 }
1713 }
1714
1715 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
1716 ShouldNotReachHere();
1717 }
1718
1719 size_t ShenandoahEvacUpdateCleanupOopStorageRootsClosure::dead_counter() const {
1720 return _dead_counter;
1721 }
1722
1723 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::reset_dead_counter() {
1724 _dead_counter = 0;
1725 }
1726
1727 // This task not only evacuates/updates marked weak roots, but also "NULL"
1728 // dead weak roots.
1729 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
1730 private:
1731 ShenandoahWeakRoot<true /*concurrent*/> _jni_roots;
1732 ShenandoahWeakRoot<true /*concurrent*/> _string_table_roots;
1733 ShenandoahWeakRoot<true /*concurrent*/> _resolved_method_table_roots;
1734 ShenandoahWeakRoot<true /*concurrent*/> _vm_roots;
1735
1736 public:
1737 ShenandoahConcurrentWeakRootsEvacUpdateTask() :
1738 AbstractGangTask("Shenandoah Concurrent Weak Root Task"),
1739 _jni_roots(OopStorageSet::jni_weak(), ShenandoahPhaseTimings::JNIWeakRoots),
1740 _string_table_roots(OopStorageSet::string_table_weak(), ShenandoahPhaseTimings::StringTableRoots),
1741 _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), ShenandoahPhaseTimings::ResolvedMethodTableRoots),
1742 _vm_roots(OopStorageSet::vm_weak(), ShenandoahPhaseTimings::VMWeakRoots) {
1743 StringTable::reset_dead_counter();
1744 ResolvedMethodTable::reset_dead_counter();
1745 }
1746
1747 ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1748 StringTable::finish_dead_counter();
1749 ResolvedMethodTable::finish_dead_counter();
1750 }
1751
1752 void work(uint worker_id) {
1753 ShenandoahEvacOOMScope oom;
1754 // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1755 // may race against OopStorage::release() calls.
1756 ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
1757 _jni_roots.oops_do(&cl, worker_id);
1758 _vm_roots.oops_do(&cl, worker_id);
1759
1760 cl.reset_dead_counter();
1761 _string_table_roots.oops_do(&cl, worker_id);
1762 StringTable::inc_dead_counter(cl.dead_counter());
1763
1764 cl.reset_dead_counter();
1765 _resolved_method_table_roots.oops_do(&cl, worker_id);
1766 ResolvedMethodTable::inc_dead_counter(cl.dead_counter());
1767 }
1768 };
1769
1770 void ShenandoahHeap::op_roots() {
1771 if (is_concurrent_root_in_progress()) {
1772 if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1773 // Concurrent weak root processing
1774 ShenandoahConcurrentWeakRootsEvacUpdateTask task;
1775 workers()->run_task(&task);
1776
1777 _unloader.unload();
1778 }
1779
1780 if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1781 ShenandoahConcurrentRootsEvacUpdateTask task(!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1782 workers()->run_task(&task);
1783 }
1784 }
1785
1786 set_concurrent_root_in_progress(false);
1787 }
1788
1789 void ShenandoahHeap::op_reset() {
1790 if (ShenandoahPacing) {
1791 pacer()->setup_for_reset();
1792 }
1793 reset_mark_bitmap();
1794 }
1795
1796 void ShenandoahHeap::op_preclean() {
1797 if (ShenandoahPacing) {
1798 pacer()->setup_for_preclean();
1799 }
1800 concurrent_mark()->preclean_weak_refs();
1801 }
1802
1803 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1804 ShenandoahMetricsSnapshot metrics;
1805 metrics.snap_before();
1806
1807 full_gc()->do_it(cause);
1808 if (UseTLAB) {
1809 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1810 resize_all_tlabs();
1811 }
1812
1813 metrics.snap_after();
1814
1815 if (metrics.is_good_progress()) {
1816 _progress_last_gc.set();
1817 } else {
1818 // Nothing to do. Tell the allocation path that we have failed to make
1819 // progress, and it can finally fail.
1820 _progress_last_gc.unset();
1821 }
1822 }
1823
1824 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1825 // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1826 // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1827 // some phase, we have to upgrade the Degenerate GC to Full GC.
1828
1829 clear_cancelled_gc();
1830
1831 ShenandoahMetricsSnapshot metrics;
1832 metrics.snap_before();
1833
1834 switch (point) {
1835 // The cases below form the Duff's-like device: it describes the actual GC cycle,
1836 // but enters it at different points, depending on which concurrent phase had
1837 // degenerated.
1838
1839 case _degenerated_outside_cycle:
1840 // We have degenerated from outside the cycle, which means something is bad with
1841 // the heap, most probably heavy humongous fragmentation, or we are very low on free
1842 // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1843 // we can do the most aggressive degen cycle, which includes processing references and
1844 // class unloading, unless those features are explicitly disabled.
1845 //
1846 // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1847 // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1848 set_process_references(heuristics()->can_process_references());
1849 set_unload_classes(heuristics()->can_unload_classes());
1850
1851 op_reset();
1852
1853 op_init_mark();
1854 if (cancelled_gc()) {
1855 op_degenerated_fail();
1856 return;
1857 }
1858
1859 case _degenerated_mark:
1860 op_final_mark();
1861 if (cancelled_gc()) {
1862 op_degenerated_fail();
1863 return;
1864 }
1865
1866 if (!has_forwarded_objects() && ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
1867 // Disarm nmethods that armed for concurrent mark. On normal cycle, it would
1868 // be disarmed while conc-roots phase is running.
1869 // TODO: Call op_conc_roots() here instead
1870 ShenandoahCodeRoots::disarm_nmethods();
1871 }
1872
1873 op_cleanup();
1874
1875 case _degenerated_evac:
1876 // If heuristics thinks we should do the cycle, this flag would be set,
1877 // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1878 if (is_evacuation_in_progress()) {
1879
1880 // Degeneration under oom-evac protocol might have left some objects in
1881 // collection set un-evacuated. Restart evacuation from the beginning to
1882 // capture all objects. For all the objects that are already evacuated,
1883 // it would be a simple check, which is supposed to be fast. This is also
1884 // safe to do even without degeneration, as CSet iterator is at beginning
1885 // in preparation for evacuation anyway.
1886 //
1887 // Before doing that, we need to make sure we never had any cset-pinned
1888 // regions. This may happen if allocation failure happened when evacuating
1889 // the about-to-be-pinned object, oom-evac protocol left the object in
1890 // the collection set, and then the pin reached the cset region. If we continue
1891 // the cycle here, we would trash the cset and alive objects in it. To avoid
1892 // it, we fail degeneration right away and slide into Full GC to recover.
1893
1894 {
1895 sync_pinned_region_status();
1896 collection_set()->clear_current_index();
1897
1898 ShenandoahHeapRegion* r;
1899 while ((r = collection_set()->next()) != NULL) {
1900 if (r->is_pinned()) {
1901 cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1902 op_degenerated_fail();
1903 return;
1904 }
1905 }
1906
1907 collection_set()->clear_current_index();
1908 }
1909
1910 op_stw_evac();
1911 if (cancelled_gc()) {
1912 op_degenerated_fail();
1913 return;
1914 }
1915 }
1916
1917 // If heuristics thinks we should do the cycle, this flag would be set,
1918 // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
1919 if (has_forwarded_objects()) {
1920 op_init_updaterefs();
1921 if (cancelled_gc()) {
1922 op_degenerated_fail();
1923 return;
1924 }
1925 }
1926
1927 case _degenerated_updaterefs:
1928 if (has_forwarded_objects()) {
1929 op_final_updaterefs();
1930 if (cancelled_gc()) {
1931 op_degenerated_fail();
1932 return;
1933 }
1934 }
1935
1936 op_cleanup();
1937 break;
1938
1939 default:
1940 ShouldNotReachHere();
1941 }
1942
1943 if (ShenandoahVerify) {
1944 verifier()->verify_after_degenerated();
1945 }
1946
1947 if (VerifyAfterGC) {
1948 Universe::verify();
1949 }
1950
1951 metrics.snap_after();
1952
1953 // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
1954 // because that probably means the heap is overloaded and/or fragmented.
1955 if (!metrics.is_good_progress()) {
1956 _progress_last_gc.unset();
1957 cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1958 op_degenerated_futile();
1959 } else {
1960 _progress_last_gc.set();
1961 }
1962 }
1963
1964 void ShenandoahHeap::op_degenerated_fail() {
1965 log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
1966 shenandoah_policy()->record_degenerated_upgrade_to_full();
1967 op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1968 }
1969
1970 void ShenandoahHeap::op_degenerated_futile() {
1971 shenandoah_policy()->record_degenerated_upgrade_to_full();
1972 op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1973 }
1974
1975 void ShenandoahHeap::force_satb_flush_all_threads() {
1976 if (!is_concurrent_mark_in_progress()) {
1977 // No need to flush SATBs
1978 return;
1979 }
1980
1981 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1982 ShenandoahThreadLocalData::set_force_satb_flush(t, true);
1983 }
1984 // The threads are not "acquiring" their thread-local data, but it does not
1985 // hurt to "release" the updates here anyway.
1986 OrderAccess::fence();
1987 }
1988
1989 void ShenandoahHeap::set_gc_state_all_threads(char state) {
1990 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1991 ShenandoahThreadLocalData::set_gc_state(t, state);
1992 }
1993 }
1994
1995 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
1996 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
1997 _gc_state.set_cond(mask, value);
1998 set_gc_state_all_threads(_gc_state.raw_value());
1999 }
2000
2001 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2002 if (has_forwarded_objects()) {
2003 set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
2004 } else {
2005 set_gc_state_mask(MARKING, in_progress);
2006 }
2007 ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2008 }
2009
2010 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2011 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2012 set_gc_state_mask(EVACUATION, in_progress);
2013 }
2014
2015 void ShenandoahHeap::set_concurrent_root_in_progress(bool in_progress) {
2016 assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2017 if (in_progress) {
2018 _concurrent_root_in_progress.set();
2019 } else {
2020 _concurrent_root_in_progress.unset();
2021 }
2022 }
2023
2024 void ShenandoahHeap::ref_processing_init() {
2025 assert(_max_workers > 0, "Sanity");
2026
2027 _ref_processor =
2028 new ReferenceProcessor(&_subject_to_discovery, // is_subject_to_discovery
2029 ParallelRefProcEnabled, // MT processing
2030 _max_workers, // Degree of MT processing
2031 true, // MT discovery
2032 _max_workers, // Degree of MT discovery
2033 false, // Reference discovery is not atomic
2034 NULL, // No closure, should be installed before use
2035 true); // Scale worker threads
2036
2037 shenandoah_assert_rp_isalive_not_installed();
2038 }
2039
2040 GCTracer* ShenandoahHeap::tracer() {
2041 return shenandoah_policy()->tracer();
2042 }
2043
2044 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2045 return _free_set->used();
2046 }
2047
2048 bool ShenandoahHeap::try_cancel_gc() {
2049 while (true) {
2050 jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2051 if (prev == CANCELLABLE) return true;
2052 else if (prev == CANCELLED) return false;
2053 assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
2054 assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
2055 if (Thread::current()->is_Java_thread()) {
2056 // We need to provide a safepoint here, otherwise we might
2057 // spin forever if a SP is pending.
2058 ThreadBlockInVM sp(JavaThread::current());
2059 SpinPause();
2060 }
2061 }
2062 }
2063
2064 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2065 if (try_cancel_gc()) {
2066 FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2067 log_info(gc)("%s", msg.buffer());
2068 Events::log(Thread::current(), "%s", msg.buffer());
2069 }
2070 }
2071
2072 uint ShenandoahHeap::max_workers() {
2073 return _max_workers;
2074 }
2075
2076 void ShenandoahHeap::stop() {
2077 // The shutdown sequence should be able to terminate when GC is running.
2078
2079 // Step 0. Notify policy to disable event recording.
2080 _shenandoah_policy->record_shutdown();
2081
2082 // Step 1. Notify control thread that we are in shutdown.
2083 // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2084 // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2085 control_thread()->prepare_for_graceful_shutdown();
2086
2087 // Step 2. Notify GC workers that we are cancelling GC.
2088 cancel_gc(GCCause::_shenandoah_stop_vm);
2089
2090 // Step 3. Wait until GC worker exits normally.
2091 control_thread()->stop();
2092
2093 // Step 4. Stop String Dedup thread if it is active
2094 if (ShenandoahStringDedup::is_enabled()) {
2095 ShenandoahStringDedup::stop();
2096 }
2097 }
2098
2099 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2100 if (!unload_classes()) return;
2101 bool purged_class;
2102
2103 // Unload classes and purge SystemDictionary.
2104 {
2105 ShenandoahGCPhase phase(full_gc ?
2106 ShenandoahPhaseTimings::full_gc_purge_class_unload :
2107 ShenandoahPhaseTimings::purge_class_unload);
2108 purged_class = SystemDictionary::do_unloading(gc_timer());
2109 }
2110
2111 {
2112 ShenandoahGCPhase phase(full_gc ?
2113 ShenandoahPhaseTimings::full_gc_purge_par :
2114 ShenandoahPhaseTimings::purge_par);
2115 ShenandoahIsAliveSelector is_alive;
2116 uint num_workers = _workers->active_workers();
2117 ShenandoahClassUnloadingTask unlink_task(is_alive.is_alive_closure(), num_workers, purged_class);
2118 _workers->run_task(&unlink_task);
2119 }
2120
2121 {
2122 ShenandoahGCPhase phase(full_gc ?
2123 ShenandoahPhaseTimings::full_gc_purge_cldg :
2124 ShenandoahPhaseTimings::purge_cldg);
2125 ClassLoaderDataGraph::purge();
2126 }
2127 // Resize and verify metaspace
2128 MetaspaceGC::compute_new_size();
2129 MetaspaceUtils::verify_metrics();
2130 }
2131
2132 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
2133 // so they should not have forwarded oops.
2134 // However, we do need to "null" dead oops in the roots, if can not be done
2135 // in concurrent cycles.
2136 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2137 ShenandoahGCPhase root_phase(full_gc ?
2138 ShenandoahPhaseTimings::full_gc_purge :
2139 ShenandoahPhaseTimings::purge);
2140 uint num_workers = _workers->active_workers();
2141 ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2142 ShenandoahPhaseTimings::full_gc_purge_par :
2143 ShenandoahPhaseTimings::purge_par;
2144 ShenandoahGCPhase phase(timing_phase);
2145 ShenandoahGCWorkerPhase worker_phase(timing_phase);
2146
2147 // Cleanup weak roots
2148 if (has_forwarded_objects()) {
2149 ShenandoahForwardedIsAliveClosure is_alive;
2150 ShenandoahUpdateRefsClosure keep_alive;
2151 ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
2152 cleaning_task(&is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2153 _workers->run_task(&cleaning_task);
2154 } else {
2155 ShenandoahIsAliveClosure is_alive;
2156 #ifdef ASSERT
2157 ShenandoahAssertNotForwardedClosure verify_cl;
2158 ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2159 cleaning_task(&is_alive, &verify_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2160 #else
2161 ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2162 cleaning_task(&is_alive, &do_nothing_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2163 #endif
2164 _workers->run_task(&cleaning_task);
2165 }
2166 }
2167
2168 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2169 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2170 stw_process_weak_roots(full_gc);
2171 if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2172 stw_unload_classes(full_gc);
2173 }
2174 }
2175
2176 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2177 set_gc_state_mask(HAS_FORWARDED, cond);
2178 }
2179
2180 void ShenandoahHeap::set_process_references(bool pr) {
2181 _process_references.set_cond(pr);
2182 }
2183
2184 void ShenandoahHeap::set_unload_classes(bool uc) {
2185 _unload_classes.set_cond(uc);
2186 }
2187
2188 bool ShenandoahHeap::process_references() const {
2189 return _process_references.is_set();
2190 }
2191
2192 bool ShenandoahHeap::unload_classes() const {
2193 return _unload_classes.is_set();
2194 }
2195
2196 address ShenandoahHeap::in_cset_fast_test_addr() {
2197 ShenandoahHeap* heap = ShenandoahHeap::heap();
2198 assert(heap->collection_set() != NULL, "Sanity");
2199 return (address) heap->collection_set()->biased_map_address();
2200 }
2201
2202 address ShenandoahHeap::cancelled_gc_addr() {
2203 return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2204 }
2205
2206 address ShenandoahHeap::gc_state_addr() {
2207 return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2208 }
2209
2210 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2211 return Atomic::load_acquire(&_bytes_allocated_since_gc_start);
2212 }
2213
2214 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2215 Atomic::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2216 }
2217
2218 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2219 _degenerated_gc_in_progress.set_cond(in_progress);
2220 }
2221
2222 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2223 _full_gc_in_progress.set_cond(in_progress);
2224 }
2225
2226 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2227 assert (is_full_gc_in_progress(), "should be");
2228 _full_gc_move_in_progress.set_cond(in_progress);
2229 }
2230
2231 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2232 set_gc_state_mask(UPDATEREFS, in_progress);
2233 }
2234
2235 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2236 ShenandoahCodeRoots::register_nmethod(nm);
2237 }
2238
2239 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2240 ShenandoahCodeRoots::unregister_nmethod(nm);
2241 }
2242
2243 void ShenandoahHeap::flush_nmethod(nmethod* nm) {
2244 ShenandoahCodeRoots::flush_nmethod(nm);
2245 }
2246
2247 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2248 heap_region_containing(o)->record_pin();
2249 return o;
2250 }
2251
2252 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2253 heap_region_containing(o)->record_unpin();
2254 }
2255
2256 void ShenandoahHeap::sync_pinned_region_status() {
2257 ShenandoahHeapLocker locker(lock());
2258
2259 for (size_t i = 0; i < num_regions(); i++) {
2260 ShenandoahHeapRegion *r = get_region(i);
2261 if (r->is_active()) {
2262 if (r->is_pinned()) {
2263 if (r->pin_count() == 0) {
2264 r->make_unpinned();
2265 }
2266 } else {
2267 if (r->pin_count() > 0) {
2268 r->make_pinned();
2269 }
2270 }
2271 }
2272 }
2273
2274 assert_pinned_region_status();
2275 }
2276
2277 #ifdef ASSERT
2278 void ShenandoahHeap::assert_pinned_region_status() {
2279 for (size_t i = 0; i < num_regions(); i++) {
2280 ShenandoahHeapRegion* r = get_region(i);
2281 assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2282 "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2283 }
2284 }
2285 #endif
2286
2287 GCTimer* ShenandoahHeap::gc_timer() const {
2288 return _gc_timer;
2289 }
2290
2291 void ShenandoahHeap::prepare_concurrent_roots() {
2292 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2293 if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
2294 set_concurrent_root_in_progress(true);
2295 }
2296 }
2297
2298 void ShenandoahHeap::prepare_concurrent_unloading() {
2299 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2300 if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2301 _unloader.prepare();
2302 }
2303 }
2304
2305 void ShenandoahHeap::finish_concurrent_unloading() {
2306 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2307 if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2308 _unloader.finish();
2309 }
2310 }
2311
2312 #ifdef ASSERT
2313 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2314 assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2315
2316 if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2317 if (UseDynamicNumberOfGCThreads) {
2318 assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2319 } else {
2320 // Use ParallelGCThreads inside safepoints
2321 assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
2322 }
2323 } else {
2324 if (UseDynamicNumberOfGCThreads) {
2325 assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2326 } else {
2327 // Use ConcGCThreads outside safepoints
2328 assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2329 }
2330 }
2331 }
2332 #endif
2333
2334 ShenandoahVerifier* ShenandoahHeap::verifier() {
2335 guarantee(ShenandoahVerify, "Should be enabled");
2336 assert (_verifier != NULL, "sanity");
2337 return _verifier;
2338 }
2339
2340 template<class T>
2341 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2342 private:
2343 T cl;
2344 ShenandoahHeap* _heap;
2345 ShenandoahRegionIterator* _regions;
2346 bool _concurrent;
2347 public:
2348 ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2349 AbstractGangTask("Concurrent Update References Task"),
2350 cl(T()),
2351 _heap(ShenandoahHeap::heap()),
2352 _regions(regions),
2353 _concurrent(concurrent) {
2354 }
2355
2356 void work(uint worker_id) {
2357 if (_concurrent) {
2358 ShenandoahConcurrentWorkerSession worker_session(worker_id);
2359 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2360 do_work();
2361 } else {
2362 ShenandoahParallelWorkerSession worker_session(worker_id);
2363 do_work();
2364 }
2365 }
2366
2367 private:
2368 void do_work() {
2369 ShenandoahHeapRegion* r = _regions->next();
2370 ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2371 while (r != NULL) {
2372 HeapWord* update_watermark = r->get_update_watermark();
2373 assert (update_watermark >= r->bottom(), "sanity");
2374 if (r->is_active() && !r->is_cset()) {
2375 _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2376 }
2377 if (ShenandoahPacing) {
2378 _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2379 }
2380 if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2381 return;
2382 }
2383 r = _regions->next();
2384 }
2385 }
2386 };
2387
2388 void ShenandoahHeap::update_heap_references(bool concurrent) {
2389 ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2390 workers()->run_task(&task);
2391 }
2392
2393 void ShenandoahHeap::op_init_updaterefs() {
2394 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2395
2396 set_evacuation_in_progress(false);
2397
2398 {
2399 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
2400 retire_and_reset_gclabs();
2401 }
2402
2403 if (ShenandoahVerify) {
2404 if (!is_degenerated_gc_in_progress()) {
2405 verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2406 }
2407 verifier()->verify_before_updaterefs();
2408 }
2409
2410 set_update_refs_in_progress(true);
2411
2412 {
2413 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_prepare);
2414
2415 make_parsable(true);
2416
2417 // Reset iterator.
2418 _update_refs_iterator.reset();
2419 }
2420
2421 if (ShenandoahPacing) {
2422 pacer()->setup_for_updaterefs();
2423 }
2424 }
2425
2426 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2427 private:
2428 ShenandoahHeapLock* const _lock;
2429
2430 public:
2431 ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2432
2433 void heap_region_do(ShenandoahHeapRegion* r) {
2434 // Drop unnecessary "pinned" state from regions that does not have CP marks
2435 // anymore, as this would allow trashing them.
2436
2437 if (r->is_active()) {
2438 if (r->is_pinned()) {
2439 if (r->pin_count() == 0) {
2440 ShenandoahHeapLocker locker(_lock);
2441 r->make_unpinned();
2442 }
2443 } else {
2444 if (r->pin_count() > 0) {
2445 ShenandoahHeapLocker locker(_lock);
2446 r->make_pinned();
2447 }
2448 }
2449 }
2450 }
2451
2452 bool is_thread_safe() { return true; }
2453 };
2454
2455 void ShenandoahHeap::op_final_updaterefs() {
2456 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2457
2458 finish_concurrent_unloading();
2459
2460 // Check if there is left-over work, and finish it
2461 if (_update_refs_iterator.has_next()) {
2462 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
2463
2464 // Finish updating references where we left off.
2465 clear_cancelled_gc();
2466 update_heap_references(false);
2467 }
2468
2469 // Clear cancelled GC, if set. On cancellation path, the block before would handle
2470 // everything. On degenerated paths, cancelled gc would not be set anyway.
2471 if (cancelled_gc()) {
2472 clear_cancelled_gc();
2473 }
2474 assert(!cancelled_gc(), "Should have been done right before");
2475
2476 if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2477 verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2478 }
2479
2480 if (is_degenerated_gc_in_progress()) {
2481 concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2482 } else {
2483 concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2484 }
2485
2486 // Has to be done before cset is clear
2487 if (ShenandoahVerify) {
2488 verifier()->verify_roots_in_to_space();
2489 }
2490
2491 {
2492 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_update_region_states);
2493 ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2494 parallel_heap_region_iterate(&cl);
2495
2496 assert_pinned_region_status();
2497 }
2498
2499 {
2500 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
2501 trash_cset_regions();
2502 }
2503
2504 set_has_forwarded_objects(false);
2505 set_update_refs_in_progress(false);
2506
2507 if (ShenandoahVerify) {
2508 verifier()->verify_after_updaterefs();
2509 }
2510
2511 if (VerifyAfterGC) {
2512 Universe::verify();
2513 }
2514
2515 {
2516 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_rebuild_freeset);
2517 ShenandoahHeapLocker locker(lock());
2518 _free_set->rebuild();
2519 }
2520 }
2521
2522 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2523 print_on(st);
2524 print_heap_regions_on(st);
2525 }
2526
2527 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2528 size_t slice = r->index() / _bitmap_regions_per_slice;
2529
2530 size_t regions_from = _bitmap_regions_per_slice * slice;
2531 size_t regions_to = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2532 for (size_t g = regions_from; g < regions_to; g++) {
2533 assert (g / _bitmap_regions_per_slice == slice, "same slice");
2534 if (skip_self && g == r->index()) continue;
2535 if (get_region(g)->is_committed()) {
2536 return true;
2537 }
2538 }
2539 return false;
2540 }
2541
2542 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2543 shenandoah_assert_heaplocked();
2544
2545 // Bitmaps in special regions do not need commits
2546 if (_bitmap_region_special) {
2547 return true;
2548 }
2549
2550 if (is_bitmap_slice_committed(r, true)) {
2551 // Some other region from the group is already committed, meaning the bitmap
2552 // slice is already committed, we exit right away.
2553 return true;
2554 }
2555
2556 // Commit the bitmap slice:
2557 size_t slice = r->index() / _bitmap_regions_per_slice;
2558 size_t off = _bitmap_bytes_per_slice * slice;
2559 size_t len = _bitmap_bytes_per_slice;
2560 if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
2561 return false;
2562 }
2563 return true;
2564 }
2565
2566 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2567 shenandoah_assert_heaplocked();
2568
2569 // Bitmaps in special regions do not need uncommits
2570 if (_bitmap_region_special) {
2571 return true;
2572 }
2573
2574 if (is_bitmap_slice_committed(r, true)) {
2575 // Some other region from the group is still committed, meaning the bitmap
2576 // slice is should stay committed, exit right away.
2577 return true;
2578 }
2579
2580 // Uncommit the bitmap slice:
2581 size_t slice = r->index() / _bitmap_regions_per_slice;
2582 size_t off = _bitmap_bytes_per_slice * slice;
2583 size_t len = _bitmap_bytes_per_slice;
2584 if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2585 return false;
2586 }
2587 return true;
2588 }
2589
2590 void ShenandoahHeap::safepoint_synchronize_begin() {
2591 if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2592 SuspendibleThreadSet::synchronize();
2593 }
2594 }
2595
2596 void ShenandoahHeap::safepoint_synchronize_end() {
2597 if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2598 SuspendibleThreadSet::desynchronize();
2599 }
2600 }
2601
2602 void ShenandoahHeap::vmop_entry_init_mark() {
2603 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2604 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2605 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark_gross);
2606
2607 try_inject_alloc_failure();
2608 VM_ShenandoahInitMark op;
2609 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2610 }
2611
2612 void ShenandoahHeap::vmop_entry_final_mark() {
2613 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2614 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2615 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark_gross);
2616
2617 try_inject_alloc_failure();
2618 VM_ShenandoahFinalMarkStartEvac op;
2619 VMThread::execute(&op); // jump to entry_final_mark under safepoint
2620 }
2621
2622 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2623 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2624 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2625 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_gross);
2626
2627 try_inject_alloc_failure();
2628 VM_ShenandoahInitUpdateRefs op;
2629 VMThread::execute(&op);
2630 }
2631
2632 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2633 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2634 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2635 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_gross);
2636
2637 try_inject_alloc_failure();
2638 VM_ShenandoahFinalUpdateRefs op;
2639 VMThread::execute(&op);
2640 }
2641
2642 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2643 TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2644 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2645 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_gross);
2646
2647 try_inject_alloc_failure();
2648 VM_ShenandoahFullGC op(cause);
2649 VMThread::execute(&op);
2650 }
2651
2652 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2653 TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2654 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2655 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_gross);
2656
2657 VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2658 VMThread::execute(°enerated_gc);
2659 }
2660
2661 void ShenandoahHeap::entry_init_mark() {
2662 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2663 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark);
2664 const char* msg = init_mark_event_message();
2665 GCTraceTime(Info, gc) time(msg, gc_timer());
2666 EventMark em("%s", msg);
2667
2668 ShenandoahWorkerScope scope(workers(),
2669 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2670 "init marking");
2671
2672 op_init_mark();
2673 }
2674
2675 void ShenandoahHeap::entry_final_mark() {
2676 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2677 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark);
2678 const char* msg = final_mark_event_message();
2679 GCTraceTime(Info, gc) time(msg, gc_timer());
2680 EventMark em("%s", msg);
2681
2682 ShenandoahWorkerScope scope(workers(),
2683 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2684 "final marking");
2685
2686 op_final_mark();
2687 }
2688
2689 void ShenandoahHeap::entry_init_updaterefs() {
2690 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2691 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs);
2692
2693 static const char* msg = "Pause Init Update Refs";
2694 GCTraceTime(Info, gc) time(msg, gc_timer());
2695 EventMark em("%s", msg);
2696
2697 // No workers used in this phase, no setup required
2698
2699 op_init_updaterefs();
2700 }
2701
2702 void ShenandoahHeap::entry_final_updaterefs() {
2703 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2704 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs);
2705
2706 static const char* msg = "Pause Final Update Refs";
2707 GCTraceTime(Info, gc) time(msg, gc_timer());
2708 EventMark em("%s", msg);
2709
2710 ShenandoahWorkerScope scope(workers(),
2711 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2712 "final reference update");
2713
2714 op_final_updaterefs();
2715 }
2716
2717 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2718 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2719 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc);
2720
2721 static const char* msg = "Pause Full";
2722 GCTraceTime(Info, gc) time(msg, gc_timer(), cause, true);
2723 EventMark em("%s", msg);
2724
2725 ShenandoahWorkerScope scope(workers(),
2726 ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2727 "full gc");
2728
2729 op_full(cause);
2730 }
2731
2732 void ShenandoahHeap::entry_degenerated(int point) {
2733 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2734 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc);
2735
2736 ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2737 const char* msg = degen_event_message(dpoint);
2738 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2739 EventMark em("%s", msg);
2740
2741 ShenandoahWorkerScope scope(workers(),
2742 ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2743 "stw degenerated gc");
2744
2745 set_degenerated_gc_in_progress(true);
2746 op_degenerated(dpoint);
2747 set_degenerated_gc_in_progress(false);
2748 }
2749
2750 void ShenandoahHeap::entry_mark() {
2751 TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2752
2753 const char* msg = conc_mark_event_message();
2754 GCTraceTime(Info, gc) time(msg);
2755 EventMark em("%s", msg);
2756
2757 ShenandoahWorkerScope scope(workers(),
2758 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2759 "concurrent marking");
2760
2761 try_inject_alloc_failure();
2762 op_mark();
2763 }
2764
2765 void ShenandoahHeap::entry_evac() {
2766 ShenandoahGCPhase conc_evac_phase(ShenandoahPhaseTimings::conc_evac);
2767 TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2768
2769 static const char* msg = "Concurrent evacuation";
2770 GCTraceTime(Info, gc) time(msg);
2771 EventMark em("%s", msg);
2772
2773 ShenandoahWorkerScope scope(workers(),
2774 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2775 "concurrent evacuation");
2776
2777 try_inject_alloc_failure();
2778 op_conc_evac();
2779 }
2780
2781 void ShenandoahHeap::entry_updaterefs() {
2782 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2783
2784 static const char* msg = "Concurrent update references";
2785 GCTraceTime(Info, gc) time(msg);
2786 EventMark em("%s", msg);
2787
2788 ShenandoahWorkerScope scope(workers(),
2789 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2790 "concurrent reference update");
2791
2792 try_inject_alloc_failure();
2793 op_updaterefs();
2794 }
2795
2796 void ShenandoahHeap::entry_roots() {
2797 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_roots);
2798
2799 static const char* msg = "Concurrent roots processing";
2800 GCTraceTime(Info, gc) time(msg);
2801 EventMark em("%s", msg);
2802
2803 ShenandoahWorkerScope scope(workers(),
2804 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2805 "concurrent root processing");
2806
2807 try_inject_alloc_failure();
2808 op_roots();
2809 }
2810
2811 void ShenandoahHeap::entry_cleanup() {
2812 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2813
2814 static const char* msg = "Concurrent cleanup";
2815 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2816 EventMark em("%s", msg);
2817
2818 // This phase does not use workers, no need for setup
2819
2820 try_inject_alloc_failure();
2821 op_cleanup();
2822 }
2823
2824 void ShenandoahHeap::entry_reset() {
2825 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2826
2827 static const char* msg = "Concurrent reset";
2828 GCTraceTime(Info, gc) time(msg);
2829 EventMark em("%s", msg);
2830
2831 ShenandoahWorkerScope scope(workers(),
2832 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2833 "concurrent reset");
2834
2835 try_inject_alloc_failure();
2836 op_reset();
2837 }
2838
2839 void ShenandoahHeap::entry_preclean() {
2840 if (ShenandoahPreclean && process_references()) {
2841 static const char* msg = "Concurrent precleaning";
2842 GCTraceTime(Info, gc) time(msg);
2843 EventMark em("%s", msg);
2844
2845 ShenandoahGCPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
2846
2847 ShenandoahWorkerScope scope(workers(),
2848 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2849 "concurrent preclean",
2850 /* check_workers = */ false);
2851
2852 try_inject_alloc_failure();
2853 op_preclean();
2854 }
2855 }
2856
2857 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2858 static const char *msg = "Concurrent uncommit";
2859 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2860 EventMark em("%s", msg);
2861
2862 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_uncommit);
2863
2864 op_uncommit(shrink_before);
2865 }
2866
2867 void ShenandoahHeap::try_inject_alloc_failure() {
2868 if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2869 _inject_alloc_failure.set();
2870 os::naked_short_sleep(1);
2871 if (cancelled_gc()) {
2872 log_info(gc)("Allocation failure was successfully injected");
2873 }
2874 }
2875 }
2876
2877 bool ShenandoahHeap::should_inject_alloc_failure() {
2878 return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2879 }
2880
2881 void ShenandoahHeap::initialize_serviceability() {
2882 _memory_pool = new ShenandoahMemoryPool(this);
2883 _cycle_memory_manager.add_pool(_memory_pool);
2884 _stw_memory_manager.add_pool(_memory_pool);
2885 }
2886
2887 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2888 GrowableArray<GCMemoryManager*> memory_managers(2);
2889 memory_managers.append(&_cycle_memory_manager);
2890 memory_managers.append(&_stw_memory_manager);
2891 return memory_managers;
2892 }
2893
2894 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2895 GrowableArray<MemoryPool*> memory_pools(1);
2896 memory_pools.append(_memory_pool);
2897 return memory_pools;
2898 }
2899
2900 MemoryUsage ShenandoahHeap::memory_usage() {
2901 return _memory_pool->get_memory_usage();
2902 }
2903
2904 void ShenandoahHeap::enter_evacuation() {
2905 _oom_evac_handler.enter_evacuation();
2906 }
2907
2908 void ShenandoahHeap::leave_evacuation() {
2909 _oom_evac_handler.leave_evacuation();
2910 }
2911
2912 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2913 _heap(ShenandoahHeap::heap()),
2914 _index(0) {}
2915
2916 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2917 _heap(heap),
2918 _index(0) {}
2919
2920 void ShenandoahRegionIterator::reset() {
2921 _index = 0;
2922 }
2923
2924 bool ShenandoahRegionIterator::has_next() const {
2925 return _index < _heap->num_regions();
2926 }
2927
2928 char ShenandoahHeap::gc_state() const {
2929 return _gc_state.raw_value();
2930 }
2931
2932 void ShenandoahHeap::deduplicate_string(oop str) {
2933 assert(java_lang_String::is_instance(str), "invariant");
2934
2935 if (ShenandoahStringDedup::is_enabled()) {
2936 ShenandoahStringDedup::deduplicate(str);
2937 }
2938 }
2939
2940 const char* ShenandoahHeap::init_mark_event_message() const {
2941 assert(!has_forwarded_objects(), "Should not have forwarded objects here");
2942
2943 bool proc_refs = process_references();
2944 bool unload_cls = unload_classes();
2945
2946 if (proc_refs && unload_cls) {
2947 return "Pause Init Mark (process weakrefs) (unload classes)";
2948 } else if (proc_refs) {
2949 return "Pause Init Mark (process weakrefs)";
2950 } else if (unload_cls) {
2951 return "Pause Init Mark (unload classes)";
2952 } else {
2953 return "Pause Init Mark";
2954 }
2955 }
2956
2957 const char* ShenandoahHeap::final_mark_event_message() const {
2958 assert(!has_forwarded_objects(), "Should not have forwarded objects here");
2959
2960 bool proc_refs = process_references();
2961 bool unload_cls = unload_classes();
2962
2963 if (proc_refs && unload_cls) {
2964 return "Pause Final Mark (process weakrefs) (unload classes)";
2965 } else if (proc_refs) {
2966 return "Pause Final Mark (process weakrefs)";
2967 } else if (unload_cls) {
2968 return "Pause Final Mark (unload classes)";
2969 } else {
2970 return "Pause Final Mark";
2971 }
2972 }
2973
2974 const char* ShenandoahHeap::conc_mark_event_message() const {
2975 assert(!has_forwarded_objects(), "Should not have forwarded objects here");
2976
2977 bool proc_refs = process_references();
2978 bool unload_cls = unload_classes();
2979
2980 if (proc_refs && unload_cls) {
2981 return "Concurrent marking (process weakrefs) (unload classes)";
2982 } else if (proc_refs) {
2983 return "Concurrent marking (process weakrefs)";
2984 } else if (unload_cls) {
2985 return "Concurrent marking (unload classes)";
2986 } else {
2987 return "Concurrent marking";
2988 }
2989 }
2990
2991 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
2992 switch (point) {
2993 case _degenerated_unset:
2994 return "Pause Degenerated GC (<UNSET>)";
2995 case _degenerated_outside_cycle:
2996 return "Pause Degenerated GC (Outside of Cycle)";
2997 case _degenerated_mark:
2998 return "Pause Degenerated GC (Mark)";
2999 case _degenerated_evac:
3000 return "Pause Degenerated GC (Evacuation)";
3001 case _degenerated_updaterefs:
3002 return "Pause Degenerated GC (Update Refs)";
3003 default:
3004 ShouldNotReachHere();
3005 return "ERROR";
3006 }
3007 }
3008
3009 jushort* ShenandoahHeap::get_liveness_cache(uint worker_id) {
3010 #ifdef ASSERT
3011 assert(_liveness_cache != NULL, "sanity");
3012 assert(worker_id < _max_workers, "sanity");
3013 for (uint i = 0; i < num_regions(); i++) {
3014 assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
3015 }
3016 #endif
3017 return _liveness_cache[worker_id];
3018 }
3019
3020 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
3021 assert(worker_id < _max_workers, "sanity");
3022 assert(_liveness_cache != NULL, "sanity");
3023 jushort* ld = _liveness_cache[worker_id];
3024 for (uint i = 0; i < num_regions(); i++) {
3025 ShenandoahHeapRegion* r = get_region(i);
3026 jushort live = ld[i];
3027 if (live > 0) {
3028 r->increase_live_data_gc_words(live);
3029 ld[i] = 0;
3030 }
3031 }
3032 }