1 /*
2 * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "memory/universe.hpp"
27
28 #include "gc/shared/gcArguments.hpp"
29 #include "gc/shared/gcTimer.hpp"
30 #include "gc/shared/gcTraceTime.inline.hpp"
31 #include "gc/shared/memAllocator.hpp"
32 #include "gc/shared/parallelCleaning.hpp"
33 #include "gc/shared/plab.hpp"
34
35 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
36 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
37 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
38 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
39 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahControlThread.hpp"
42 #include "gc/shenandoah/shenandoahFreeSet.hpp"
43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
44 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
45 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
46 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
47 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
48 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
49 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
50 #include "gc/shenandoah/shenandoahMetrics.hpp"
51 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
52 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
53 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
54 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
55 #include "gc/shenandoah/shenandoahStringDedup.hpp"
56 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
57 #include "gc/shenandoah/shenandoahUtils.hpp"
58 #include "gc/shenandoah/shenandoahVerifier.hpp"
59 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
60 #include "gc/shenandoah/shenandoahVMOperations.hpp"
61 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
62 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
63 #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
64 #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp"
65 #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp"
66 #include "gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp"
67 #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp"
68 #include "gc/shenandoah/heuristics/shenandoahTraversalHeuristics.hpp"
69 #if INCLUDE_JFR
70 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
71 #endif
72
73 #include "memory/metaspace.hpp"
74 #include "oops/compressedOops.inline.hpp"
75 #include "runtime/globals.hpp"
76 #include "runtime/interfaceSupport.inline.hpp"
77 #include "runtime/safepointMechanism.hpp"
78 #include "runtime/vmThread.hpp"
79 #include "services/mallocTracker.hpp"
80
81 #ifdef ASSERT
82 template <class T>
83 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
84 T o = RawAccess<>::oop_load(p);
85 if (! CompressedOops::is_null(o)) {
86 oop obj = CompressedOops::decode_not_null(o);
87 shenandoah_assert_not_forwarded(p, obj);
88 }
89 }
90
91 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
92 void ShenandoahAssertToSpaceClosure::do_oop(oop* p) { do_oop_work(p); }
93 #endif
94
95 class ShenandoahPretouchHeapTask : public AbstractGangTask {
96 private:
97 ShenandoahRegionIterator _regions;
98 const size_t _page_size;
99 public:
100 ShenandoahPretouchHeapTask(size_t page_size) :
101 AbstractGangTask("Shenandoah Pretouch Heap"),
102 _page_size(page_size) {}
103
104 virtual void work(uint worker_id) {
105 ShenandoahHeapRegion* r = _regions.next();
106 while (r != NULL) {
107 os::pretouch_memory(r->bottom(), r->end(), _page_size);
108 r = _regions.next();
109 }
110 }
111 };
112
113 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
114 private:
115 ShenandoahRegionIterator _regions;
116 char* _bitmap_base;
117 const size_t _bitmap_size;
118 const size_t _page_size;
119 public:
120 ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
121 AbstractGangTask("Shenandoah Pretouch Bitmap"),
122 _bitmap_base(bitmap_base),
123 _bitmap_size(bitmap_size),
124 _page_size(page_size) {}
125
126 virtual void work(uint worker_id) {
127 ShenandoahHeapRegion* r = _regions.next();
128 while (r != NULL) {
129 size_t start = r->region_number() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
130 size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
131 assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
132
133 os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
134
135 r = _regions.next();
136 }
137 }
138 };
139
140 jint ShenandoahHeap::initialize() {
141 initialize_heuristics();
142
143 //
144 // Figure out heap sizing
145 //
146
147 size_t init_byte_size = InitialHeapSize;
148 size_t min_byte_size = MinHeapSize;
149 size_t max_byte_size = MaxHeapSize;
150 size_t heap_alignment = HeapAlignment;
151
152 size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
153
154 if (ShenandoahAlwaysPreTouch) {
155 // Enabled pre-touch means the entire heap is committed right away.
156 init_byte_size = max_byte_size;
157 }
158
159 Universe::check_alignment(max_byte_size, reg_size_bytes, "Shenandoah heap");
160 Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
161
162 _num_regions = ShenandoahHeapRegion::region_count();
163
164 size_t num_committed_regions = init_byte_size / reg_size_bytes;
165 num_committed_regions = MIN2(num_committed_regions, _num_regions);
166 assert(num_committed_regions <= _num_regions, "sanity");
167 _initial_size = num_committed_regions * reg_size_bytes;
168
169 size_t num_min_regions = min_byte_size / reg_size_bytes;
170 num_min_regions = MIN2(num_min_regions, _num_regions);
171 assert(num_min_regions <= _num_regions, "sanity");
172 _minimum_size = num_min_regions * reg_size_bytes;
173
174 _committed = _initial_size;
175
176 size_t heap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
177 size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
178
179 //
180 // Reserve and commit memory for heap
181 //
182
183 ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
184 initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size()));
185 _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
186 _heap_region_special = heap_rs.special();
187
188 assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
189 "Misaligned heap: " PTR_FORMAT, p2i(base()));
190
191 #if SHENANDOAH_OPTIMIZED_OBJTASK
192 // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
193 // Fail if we ever attempt to address more than we can.
194 if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
195 FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
196 "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
197 "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
198 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
199 vm_exit_during_initialization("Fatal Error", buf);
200 }
201 #endif
202
203 ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
204 if (!_heap_region_special) {
205 os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
206 "Cannot commit heap memory");
207 }
208
209 //
210 // Reserve and commit memory for bitmap(s)
211 //
212
213 _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
214 _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
215
216 size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
217
218 guarantee(bitmap_bytes_per_region != 0,
219 "Bitmap bytes per region should not be zero");
220 guarantee(is_power_of_2(bitmap_bytes_per_region),
221 "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
222
223 if (bitmap_page_size > bitmap_bytes_per_region) {
224 _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
225 _bitmap_bytes_per_slice = bitmap_page_size;
226 } else {
227 _bitmap_regions_per_slice = 1;
228 _bitmap_bytes_per_slice = bitmap_bytes_per_region;
229 }
230
231 guarantee(_bitmap_regions_per_slice >= 1,
232 "Should have at least one region per slice: " SIZE_FORMAT,
233 _bitmap_regions_per_slice);
234
235 guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
236 "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
237 _bitmap_bytes_per_slice, bitmap_page_size);
238
239 ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
240 MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
241 _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
242 _bitmap_region_special = bitmap.special();
243
244 size_t bitmap_init_commit = _bitmap_bytes_per_slice *
245 align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
246 bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
247 if (!_bitmap_region_special) {
248 os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
249 "Cannot commit bitmap memory");
250 }
251
252 _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
253
254 if (ShenandoahVerify) {
255 ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
256 if (!verify_bitmap.special()) {
257 os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
258 "Cannot commit verification bitmap memory");
259 }
260 MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
261 MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
262 _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
263 _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
264 }
265
266 // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
267 ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
268 MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
269 _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
270 _aux_bitmap_region_special = aux_bitmap.special();
271 _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
272
273 //
274 // Create regions and region sets
275 //
276
277 _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
278 _free_set = new ShenandoahFreeSet(this, _num_regions);
279 _collection_set = new ShenandoahCollectionSet(this, sh_rs.base(), sh_rs.size());
280
281 {
282 ShenandoahHeapLocker locker(lock());
283
284 size_t size_words = ShenandoahHeapRegion::region_size_words();
285
286 for (size_t i = 0; i < _num_regions; i++) {
287 HeapWord* start = (HeapWord*)sh_rs.base() + size_words * i;
288 bool is_committed = i < num_committed_regions;
289 ShenandoahHeapRegion* r = new ShenandoahHeapRegion(this, start, size_words, i, is_committed);
290
291 _marking_context->initialize_top_at_mark_start(r);
292 _regions[i] = r;
293 assert(!collection_set()->is_in(i), "New region should not be in collection set");
294 }
295
296 // Initialize to complete
297 _marking_context->mark_complete();
298
299 _free_set->rebuild();
300 }
301
302 if (ShenandoahAlwaysPreTouch) {
303 assert(!AlwaysPreTouch, "Should have been overridden");
304
305 // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
306 // before initialize() below zeroes it with initializing thread. For any given region,
307 // we touch the region and the corresponding bitmaps from the same thread.
308 ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
309
310 size_t pretouch_heap_page_size = heap_page_size;
311 size_t pretouch_bitmap_page_size = bitmap_page_size;
312
313 #ifdef LINUX
314 // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
315 // pages. But, the kernel needs to know that every small page is used, in order to coalesce
316 // them into huge one. Therefore, we need to pretouch with smaller pages.
317 if (UseTransparentHugePages) {
318 pretouch_heap_page_size = (size_t)os::vm_page_size();
319 pretouch_bitmap_page_size = (size_t)os::vm_page_size();
320 }
321 #endif
322
323 // OS memory managers may want to coalesce back-to-back pages. Make their jobs
324 // simpler by pre-touching continuous spaces (heap and bitmap) separately.
325
326 log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
327 _num_regions, pretouch_bitmap_page_size);
328 ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size);
329 _workers->run_task(&bcl);
330
331 log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
332 _num_regions, pretouch_heap_page_size);
333 ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size);
334 _workers->run_task(&hcl);
335 }
336
337 //
338 // Initialize the rest of GC subsystems
339 //
340
341 _liveness_cache = NEW_C_HEAP_ARRAY(jushort*, _max_workers, mtGC);
342 for (uint worker = 0; worker < _max_workers; worker++) {
343 _liveness_cache[worker] = NEW_C_HEAP_ARRAY(jushort, _num_regions, mtGC);
344 Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(jushort));
345 }
346
347 // The call below uses stuff (the SATB* things) that are in G1, but probably
348 // belong into a shared location.
349 ShenandoahBarrierSet::satb_mark_queue_set().initialize(this,
350 20 /* G1SATBProcessCompletedThreshold */,
351 60 /* G1SATBBufferEnqueueingThresholdPercent */);
352
353 _monitoring_support = new ShenandoahMonitoringSupport(this);
354 _phase_timings = new ShenandoahPhaseTimings();
355 ShenandoahStringDedup::initialize();
356 ShenandoahCodeRoots::initialize();
357
358 if (ShenandoahAllocationTrace) {
359 _alloc_tracker = new ShenandoahAllocTracker();
360 }
361
362 if (ShenandoahPacing) {
363 _pacer = new ShenandoahPacer(this);
364 _pacer->setup_for_idle();
365 } else {
366 _pacer = NULL;
367 }
368
369 _traversal_gc = heuristics()->can_do_traversal_gc() ?
370 new ShenandoahTraversalGC(this, _num_regions) :
371 NULL;
372
373 _control_thread = new ShenandoahControlThread();
374
375 log_info(gc, init)("Initialize Shenandoah heap: " SIZE_FORMAT "%s initial, " SIZE_FORMAT "%s min, " SIZE_FORMAT "%s max",
376 byte_size_in_proper_unit(_initial_size), proper_unit_for_byte_size(_initial_size),
377 byte_size_in_proper_unit(_minimum_size), proper_unit_for_byte_size(_minimum_size),
378 byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity())
379 );
380
381 log_info(gc, init)("Safepointing mechanism: %s",
382 SafepointMechanism::uses_thread_local_poll() ? "thread-local poll" :
383 (SafepointMechanism::uses_global_page_poll() ? "global-page poll" : "unknown"));
384
385 return JNI_OK;
386 }
387
388 void ShenandoahHeap::initialize_heuristics() {
389 if (ShenandoahGCHeuristics != NULL) {
390 if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) {
391 _heuristics = new ShenandoahAggressiveHeuristics();
392 } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) {
393 _heuristics = new ShenandoahStaticHeuristics();
394 } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) {
395 _heuristics = new ShenandoahAdaptiveHeuristics();
396 } else if (strcmp(ShenandoahGCHeuristics, "passive") == 0) {
397 _heuristics = new ShenandoahPassiveHeuristics();
398 } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) {
399 _heuristics = new ShenandoahCompactHeuristics();
400 } else if (strcmp(ShenandoahGCHeuristics, "traversal") == 0) {
401 _heuristics = new ShenandoahTraversalHeuristics();
402 } else {
403 vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option");
404 }
405
406 if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
407 vm_exit_during_initialization(
408 err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
409 _heuristics->name()));
410 }
411 if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
412 vm_exit_during_initialization(
413 err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
414 _heuristics->name()));
415 }
416 log_info(gc, init)("Shenandoah heuristics: %s",
417 _heuristics->name());
418 } else {
419 ShouldNotReachHere();
420 }
421
422 }
423
424 #ifdef _MSC_VER
425 #pragma warning( push )
426 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
427 #endif
428
429 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
430 CollectedHeap(),
431 _initial_size(0),
432 _used(0),
433 _committed(0),
434 _bytes_allocated_since_gc_start(0),
435 _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
436 _workers(NULL),
437 _safepoint_workers(NULL),
438 _heap_region_special(false),
439 _num_regions(0),
440 _regions(NULL),
441 _update_refs_iterator(this),
442 _control_thread(NULL),
443 _shenandoah_policy(policy),
444 _heuristics(NULL),
445 _free_set(NULL),
446 _scm(new ShenandoahConcurrentMark()),
447 _traversal_gc(NULL),
448 _full_gc(new ShenandoahMarkCompact()),
449 _pacer(NULL),
450 _verifier(NULL),
451 _alloc_tracker(NULL),
452 _phase_timings(NULL),
453 _monitoring_support(NULL),
454 _memory_pool(NULL),
455 _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
456 _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
457 _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
458 _soft_ref_policy(),
459 _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
460 _ref_processor(NULL),
461 _marking_context(NULL),
462 _bitmap_size(0),
463 _bitmap_regions_per_slice(0),
464 _bitmap_bytes_per_slice(0),
465 _bitmap_region_special(false),
466 _aux_bitmap_region_special(false),
467 _liveness_cache(NULL),
468 _collection_set(NULL)
469 {
470 log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
471 log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial");
472
473 BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
474
475 _max_workers = MAX2(_max_workers, 1U);
476 _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
477 /* are_GC_task_threads */ true,
478 /* are_ConcurrentGC_threads */ true);
479 if (_workers == NULL) {
480 vm_exit_during_initialization("Failed necessary allocation.");
481 } else {
482 _workers->initialize_workers();
483 }
484
485 if (ShenandoahParallelSafepointThreads > 1) {
486 _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
487 ShenandoahParallelSafepointThreads,
488 /* are_GC_task_threads */ false,
489 /* are_ConcurrentGC_threads */ false);
490 _safepoint_workers->initialize_workers();
491 }
492 }
493
494 #ifdef _MSC_VER
495 #pragma warning( pop )
496 #endif
497
498 class ShenandoahResetBitmapTask : public AbstractGangTask {
499 private:
500 ShenandoahRegionIterator _regions;
501
502 public:
503 ShenandoahResetBitmapTask() :
504 AbstractGangTask("Parallel Reset Bitmap Task") {}
505
506 void work(uint worker_id) {
507 ShenandoahHeapRegion* region = _regions.next();
508 ShenandoahHeap* heap = ShenandoahHeap::heap();
509 ShenandoahMarkingContext* const ctx = heap->marking_context();
510 while (region != NULL) {
511 if (heap->is_bitmap_slice_committed(region)) {
512 ctx->clear_bitmap(region);
513 }
514 region = _regions.next();
515 }
516 }
517 };
518
519 void ShenandoahHeap::reset_mark_bitmap() {
520 assert_gc_workers(_workers->active_workers());
521 mark_incomplete_marking_context();
522
523 ShenandoahResetBitmapTask task;
524 _workers->run_task(&task);
525 }
526
527 void ShenandoahHeap::print_on(outputStream* st) const {
528 st->print_cr("Shenandoah Heap");
529 st->print_cr(" " SIZE_FORMAT "K total, " SIZE_FORMAT "K committed, " SIZE_FORMAT "K used",
530 max_capacity() / K, committed() / K, used() / K);
531 st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"K regions",
532 num_regions(), ShenandoahHeapRegion::region_size_bytes() / K);
533
534 st->print("Status: ");
535 if (has_forwarded_objects()) st->print("has forwarded objects, ");
536 if (is_concurrent_mark_in_progress()) st->print("marking, ");
537 if (is_evacuation_in_progress()) st->print("evacuating, ");
538 if (is_update_refs_in_progress()) st->print("updating refs, ");
539 if (is_concurrent_traversal_in_progress()) st->print("traversal, ");
540 if (is_degenerated_gc_in_progress()) st->print("degenerated gc, ");
541 if (is_full_gc_in_progress()) st->print("full gc, ");
542 if (is_full_gc_move_in_progress()) st->print("full gc move, ");
543
544 if (cancelled_gc()) {
545 st->print("cancelled");
546 } else {
547 st->print("not cancelled");
548 }
549 st->cr();
550
551 st->print_cr("Reserved region:");
552 st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
553 p2i(reserved_region().start()),
554 p2i(reserved_region().end()));
555
556 ShenandoahCollectionSet* cset = collection_set();
557 st->print_cr("Collection set:");
558 if (cset != NULL) {
559 st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
560 st->print_cr(" - map (biased): " PTR_FORMAT, p2i(cset->biased_map_address()));
561 } else {
562 st->print_cr(" (NULL)");
563 }
564
565 st->cr();
566 MetaspaceUtils::print_on(st);
567
568 if (Verbose) {
569 print_heap_regions_on(st);
570 }
571 }
572
573 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
574 public:
575 void do_thread(Thread* thread) {
576 assert(thread != NULL, "Sanity");
577 assert(thread->is_Worker_thread(), "Only worker thread expected");
578 ShenandoahThreadLocalData::initialize_gclab(thread);
579 }
580 };
581
582 void ShenandoahHeap::post_initialize() {
583 CollectedHeap::post_initialize();
584 MutexLocker ml(Threads_lock);
585
586 ShenandoahInitWorkerGCLABClosure init_gclabs;
587 _workers->threads_do(&init_gclabs);
588
589 // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
590 // Now, we will let WorkGang to initialize gclab when new worker is created.
591 _workers->set_initialize_gclab();
592
593 _scm->initialize(_max_workers);
594 _full_gc->initialize(_gc_timer);
595
596 ref_processing_init();
597
598 _heuristics->initialize();
599
600 JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
601 }
602
603 size_t ShenandoahHeap::used() const {
604 return OrderAccess::load_acquire(&_used);
605 }
606
607 size_t ShenandoahHeap::committed() const {
608 OrderAccess::acquire();
609 return _committed;
610 }
611
612 void ShenandoahHeap::increase_committed(size_t bytes) {
613 assert_heaplock_or_safepoint();
614 _committed += bytes;
615 }
616
617 void ShenandoahHeap::decrease_committed(size_t bytes) {
618 assert_heaplock_or_safepoint();
619 _committed -= bytes;
620 }
621
622 void ShenandoahHeap::increase_used(size_t bytes) {
623 Atomic::add(bytes, &_used);
624 }
625
626 void ShenandoahHeap::set_used(size_t bytes) {
627 OrderAccess::release_store_fence(&_used, bytes);
628 }
629
630 void ShenandoahHeap::decrease_used(size_t bytes) {
631 assert(used() >= bytes, "never decrease heap size by more than we've left");
632 Atomic::sub(bytes, &_used);
633 }
634
635 void ShenandoahHeap::increase_allocated(size_t bytes) {
636 Atomic::add(bytes, &_bytes_allocated_since_gc_start);
637 }
638
639 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
640 size_t bytes = words * HeapWordSize;
641 if (!waste) {
642 increase_used(bytes);
643 }
644 increase_allocated(bytes);
645 if (ShenandoahPacing) {
646 control_thread()->pacing_notify_alloc(words);
647 if (waste) {
648 pacer()->claim_for_alloc(words, true);
649 }
650 }
651 }
652
653 size_t ShenandoahHeap::capacity() const {
654 return committed();
655 }
656
657 size_t ShenandoahHeap::max_capacity() const {
658 return _num_regions * ShenandoahHeapRegion::region_size_bytes();
659 }
660
661 size_t ShenandoahHeap::min_capacity() const {
662 return _minimum_size;
663 }
664
665 size_t ShenandoahHeap::initial_capacity() const {
666 return _initial_size;
667 }
668
669 bool ShenandoahHeap::is_in(const void* p) const {
670 HeapWord* heap_base = (HeapWord*) base();
671 HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
672 return p >= heap_base && p < last_region_end;
673 }
674
675 void ShenandoahHeap::op_uncommit(double shrink_before) {
676 assert (ShenandoahUncommit, "should be enabled");
677
678 // Application allocates from the beginning of the heap, and GC allocates at
679 // the end of it. It is more efficient to uncommit from the end, so that applications
680 // could enjoy the near committed regions. GC allocations are much less frequent,
681 // and therefore can accept the committing costs.
682
683 size_t count = 0;
684 for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
685 ShenandoahHeapRegion* r = get_region(i - 1);
686 if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
687 ShenandoahHeapLocker locker(lock());
688 if (r->is_empty_committed()) {
689 // Do not uncommit below minimal capacity
690 if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
691 break;
692 }
693
694 r->make_uncommitted();
695 count++;
696 }
697 }
698 SpinPause(); // allow allocators to take the lock
699 }
700
701 if (count > 0) {
702 control_thread()->notify_heap_changed();
703 }
704 }
705
706 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
707 // New object should fit the GCLAB size
708 size_t min_size = MAX2(size, PLAB::min_size());
709
710 // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
711 size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
712 new_size = MIN2(new_size, PLAB::max_size());
713 new_size = MAX2(new_size, PLAB::min_size());
714
715 // Record new heuristic value even if we take any shortcut. This captures
716 // the case when moderately-sized objects always take a shortcut. At some point,
717 // heuristics should catch up with them.
718 ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
719
720 if (new_size < size) {
721 // New size still does not fit the object. Fall back to shared allocation.
722 // This avoids retiring perfectly good GCLABs, when we encounter a large object.
723 return NULL;
724 }
725
726 // Retire current GCLAB, and allocate a new one.
727 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
728 gclab->retire();
729
730 size_t actual_size = 0;
731 HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
732 if (gclab_buf == NULL) {
733 return NULL;
734 }
735
736 assert (size <= actual_size, "allocation should fit");
737
738 if (ZeroTLAB) {
739 // ..and clear it.
740 Copy::zero_to_words(gclab_buf, actual_size);
741 } else {
742 // ...and zap just allocated object.
743 #ifdef ASSERT
744 // Skip mangling the space corresponding to the object header to
745 // ensure that the returned space is not considered parsable by
746 // any concurrent GC thread.
747 size_t hdr_size = oopDesc::header_size();
748 Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
749 #endif // ASSERT
750 }
751 gclab->set_buf(gclab_buf, actual_size);
752 return gclab->allocate(size);
753 }
754
755 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
756 size_t requested_size,
757 size_t* actual_size) {
758 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
759 HeapWord* res = allocate_memory(req);
760 if (res != NULL) {
761 *actual_size = req.actual_size();
762 } else {
763 *actual_size = 0;
764 }
765 return res;
766 }
767
768 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
769 size_t word_size,
770 size_t* actual_size) {
771 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
772 HeapWord* res = allocate_memory(req);
773 if (res != NULL) {
774 *actual_size = req.actual_size();
775 } else {
776 *actual_size = 0;
777 }
778 return res;
779 }
780
781 ShenandoahHeap* ShenandoahHeap::heap() {
782 CollectedHeap* heap = Universe::heap();
783 assert(heap != NULL, "Unitialized access to ShenandoahHeap::heap()");
784 assert(heap->kind() == CollectedHeap::Shenandoah, "not a shenandoah heap");
785 return (ShenandoahHeap*) heap;
786 }
787
788 ShenandoahHeap* ShenandoahHeap::heap_no_check() {
789 CollectedHeap* heap = Universe::heap();
790 return (ShenandoahHeap*) heap;
791 }
792
793 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
794 ShenandoahAllocTrace trace_alloc(req.size(), req.type());
795
796 intptr_t pacer_epoch = 0;
797 bool in_new_region = false;
798 HeapWord* result = NULL;
799
800 if (req.is_mutator_alloc()) {
801 if (ShenandoahPacing) {
802 pacer()->pace_for_alloc(req.size());
803 pacer_epoch = pacer()->epoch();
804 }
805
806 if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
807 result = allocate_memory_under_lock(req, in_new_region);
808 }
809
810 // Allocation failed, block until control thread reacted, then retry allocation.
811 //
812 // It might happen that one of the threads requesting allocation would unblock
813 // way later after GC happened, only to fail the second allocation, because
814 // other threads have already depleted the free storage. In this case, a better
815 // strategy is to try again, as long as GC makes progress.
816 //
817 // Then, we need to make sure the allocation was retried after at least one
818 // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
819
820 size_t tries = 0;
821
822 while (result == NULL && _progress_last_gc.is_set()) {
823 tries++;
824 control_thread()->handle_alloc_failure(req.size());
825 result = allocate_memory_under_lock(req, in_new_region);
826 }
827
828 while (result == NULL && tries <= ShenandoahFullGCThreshold) {
829 tries++;
830 control_thread()->handle_alloc_failure(req.size());
831 result = allocate_memory_under_lock(req, in_new_region);
832 }
833
834 } else {
835 assert(req.is_gc_alloc(), "Can only accept GC allocs here");
836 result = allocate_memory_under_lock(req, in_new_region);
837 // Do not call handle_alloc_failure() here, because we cannot block.
838 // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
839 }
840
841 if (in_new_region) {
842 control_thread()->notify_heap_changed();
843 }
844
845 if (result != NULL) {
846 size_t requested = req.size();
847 size_t actual = req.actual_size();
848
849 assert (req.is_lab_alloc() || (requested == actual),
850 "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
851 ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
852
853 if (req.is_mutator_alloc()) {
854 notify_mutator_alloc_words(actual, false);
855
856 // If we requested more than we were granted, give the rest back to pacer.
857 // This only matters if we are in the same pacing epoch: do not try to unpace
858 // over the budget for the other phase.
859 if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
860 pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
861 }
862 } else {
863 increase_used(actual*HeapWordSize);
864 }
865 }
866
867 return result;
868 }
869
870 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
871 ShenandoahHeapLocker locker(lock());
872 return _free_set->allocate(req, in_new_region);
873 }
874
875 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
876 bool* gc_overhead_limit_was_exceeded) {
877 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
878 return allocate_memory(req);
879 }
880
881 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
882 size_t size,
883 Metaspace::MetadataType mdtype) {
884 MetaWord* result;
885
886 // Inform metaspace OOM to GC heuristics if class unloading is possible.
887 if (heuristics()->can_unload_classes()) {
888 ShenandoahHeuristics* h = heuristics();
889 h->record_metaspace_oom();
890 }
891
892 // Expand and retry allocation
893 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
894 if (result != NULL) {
895 return result;
896 }
897
898 // Start full GC
899 collect(GCCause::_metadata_GC_clear_soft_refs);
900
901 // Retry allocation
902 result = loader_data->metaspace_non_null()->allocate(size, mdtype);
903 if (result != NULL) {
904 return result;
905 }
906
907 // Expand and retry allocation
908 result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
909 if (result != NULL) {
910 return result;
911 }
912
913 // Out of memory
914 return NULL;
915 }
916
917 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
918 private:
919 ShenandoahHeap* const _heap;
920 Thread* const _thread;
921 public:
922 ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
923 _heap(heap), _thread(Thread::current()) {}
924
925 void do_object(oop p) {
926 shenandoah_assert_marked(NULL, p);
927 if (!p->is_forwarded()) {
928 _heap->evacuate_object(p, _thread);
929 }
930 }
931 };
932
933 class ShenandoahEvacuationTask : public AbstractGangTask {
934 private:
935 ShenandoahHeap* const _sh;
936 ShenandoahCollectionSet* const _cs;
937 bool _concurrent;
938 public:
939 ShenandoahEvacuationTask(ShenandoahHeap* sh,
940 ShenandoahCollectionSet* cs,
941 bool concurrent) :
942 AbstractGangTask("Parallel Evacuation Task"),
943 _sh(sh),
944 _cs(cs),
945 _concurrent(concurrent)
946 {}
947
948 void work(uint worker_id) {
949 if (_concurrent) {
950 ShenandoahConcurrentWorkerSession worker_session(worker_id);
951 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
952 ShenandoahEvacOOMScope oom_evac_scope;
953 do_work();
954 } else {
955 ShenandoahParallelWorkerSession worker_session(worker_id);
956 ShenandoahEvacOOMScope oom_evac_scope;
957 do_work();
958 }
959 }
960
961 private:
962 void do_work() {
963 ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
964 ShenandoahHeapRegion* r;
965 while ((r =_cs->claim_next()) != NULL) {
966 assert(r->has_live(), "all-garbage regions are reclaimed early");
967 _sh->marked_object_iterate(r, &cl);
968
969 if (ShenandoahPacing) {
970 _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
971 }
972
973 if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
974 break;
975 }
976 }
977 }
978 };
979
980 void ShenandoahHeap::trash_cset_regions() {
981 ShenandoahHeapLocker locker(lock());
982
983 ShenandoahCollectionSet* set = collection_set();
984 ShenandoahHeapRegion* r;
985 set->clear_current_index();
986 while ((r = set->next()) != NULL) {
987 r->make_trash();
988 }
989 collection_set()->clear();
990 }
991
992 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
993 st->print_cr("Heap Regions:");
994 st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
995 st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
996 st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start (previous, next)");
997 st->print_cr("SN=alloc sequence numbers (first mutator, last mutator, first gc, last gc)");
998
999 for (size_t i = 0; i < num_regions(); i++) {
1000 get_region(i)->print_on(st);
1001 }
1002 }
1003
1004 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1005 assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1006
1007 oop humongous_obj = oop(start->bottom());
1008 size_t size = humongous_obj->size();
1009 size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1010 size_t index = start->region_number() + required_regions - 1;
1011
1012 assert(!start->has_live(), "liveness must be zero");
1013
1014 for(size_t i = 0; i < required_regions; i++) {
1015 // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1016 // as it expects that every region belongs to a humongous region starting with a humongous start region.
1017 ShenandoahHeapRegion* region = get_region(index --);
1018
1019 assert(region->is_humongous(), "expect correct humongous start or continuation");
1020 assert(!region->is_cset(), "Humongous region should not be in collection set");
1021
1022 region->make_trash_immediate();
1023 }
1024 }
1025
1026 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1027 public:
1028 void do_thread(Thread* thread) {
1029 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1030 assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1031 gclab->retire();
1032 }
1033 };
1034
1035 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1036 if (UseTLAB) {
1037 CollectedHeap::ensure_parsability(retire_tlabs);
1038 }
1039 ShenandoahRetireGCLABClosure cl;
1040 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1041 cl.do_thread(t);
1042 }
1043 workers()->threads_do(&cl);
1044 }
1045
1046 void ShenandoahHeap::resize_tlabs() {
1047 CollectedHeap::resize_all_tlabs();
1048 }
1049
1050 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1051 private:
1052 ShenandoahRootEvacuator* _rp;
1053
1054 public:
1055 ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1056 AbstractGangTask("Shenandoah evacuate and update roots"),
1057 _rp(rp) {}
1058
1059 void work(uint worker_id) {
1060 ShenandoahParallelWorkerSession worker_session(worker_id);
1061 ShenandoahEvacOOMScope oom_evac_scope;
1062 ShenandoahEvacuateUpdateRootsClosure cl;
1063 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1064 _rp->roots_do(worker_id, &cl);
1065 }
1066 };
1067
1068 void ShenandoahHeap::evacuate_and_update_roots() {
1069 #if COMPILER2_OR_JVMCI
1070 DerivedPointerTable::clear();
1071 #endif
1072 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1073
1074 {
1075 ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
1076 ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1077 workers()->run_task(&roots_task);
1078 }
1079
1080 #if COMPILER2_OR_JVMCI
1081 DerivedPointerTable::update_pointers();
1082 #endif
1083 }
1084
1085 // Returns size in bytes
1086 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1087 if (ShenandoahElasticTLAB) {
1088 // With Elastic TLABs, return the max allowed size, and let the allocation path
1089 // figure out the safe size for current allocation.
1090 return ShenandoahHeapRegion::max_tlab_size_bytes();
1091 } else {
1092 return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1093 }
1094 }
1095
1096 size_t ShenandoahHeap::max_tlab_size() const {
1097 // Returns size in words
1098 return ShenandoahHeapRegion::max_tlab_size_words();
1099 }
1100
1101 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1102 public:
1103 void do_thread(Thread* thread) {
1104 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1105 gclab->retire();
1106 if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1107 ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1108 }
1109 }
1110 };
1111
1112 void ShenandoahHeap::retire_and_reset_gclabs() {
1113 ShenandoahRetireAndResetGCLABClosure cl;
1114 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1115 cl.do_thread(t);
1116 }
1117 workers()->threads_do(&cl);
1118 }
1119
1120 void ShenandoahHeap::collect(GCCause::Cause cause) {
1121 control_thread()->request_gc(cause);
1122 }
1123
1124 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1125 //assert(false, "Shouldn't need to do full collections");
1126 }
1127
1128 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1129 Space* sp = heap_region_containing(addr);
1130 if (sp != NULL) {
1131 return sp->block_start(addr);
1132 }
1133 return NULL;
1134 }
1135
1136 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1137 Space* sp = heap_region_containing(addr);
1138 return sp->block_is_obj(addr);
1139 }
1140
1141 jlong ShenandoahHeap::millis_since_last_gc() {
1142 double v = heuristics()->time_since_last_gc() * 1000;
1143 assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1144 return (jlong)v;
1145 }
1146
1147 void ShenandoahHeap::prepare_for_verify() {
1148 if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1149 make_parsable(false);
1150 }
1151 }
1152
1153 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1154 workers()->print_worker_threads_on(st);
1155 if (ShenandoahStringDedup::is_enabled()) {
1156 ShenandoahStringDedup::print_worker_threads_on(st);
1157 }
1158 }
1159
1160 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1161 workers()->threads_do(tcl);
1162 if (_safepoint_workers != NULL) {
1163 _safepoint_workers->threads_do(tcl);
1164 }
1165 if (ShenandoahStringDedup::is_enabled()) {
1166 ShenandoahStringDedup::threads_do(tcl);
1167 }
1168 }
1169
1170 void ShenandoahHeap::print_tracing_info() const {
1171 LogTarget(Info, gc, stats) lt;
1172 if (lt.is_enabled()) {
1173 ResourceMark rm;
1174 LogStream ls(lt);
1175
1176 phase_timings()->print_on(&ls);
1177
1178 ls.cr();
1179 ls.cr();
1180
1181 shenandoah_policy()->print_gc_stats(&ls);
1182
1183 ls.cr();
1184 ls.cr();
1185
1186 if (ShenandoahPacing) {
1187 pacer()->print_on(&ls);
1188 }
1189
1190 ls.cr();
1191 ls.cr();
1192
1193 if (ShenandoahAllocationTrace) {
1194 assert(alloc_tracker() != NULL, "Must be");
1195 alloc_tracker()->print_on(&ls);
1196 } else {
1197 ls.print_cr(" Allocation tracing is disabled, use -XX:+ShenandoahAllocationTrace to enable.");
1198 }
1199 }
1200 }
1201
1202 void ShenandoahHeap::verify(VerifyOption vo) {
1203 if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1204 if (ShenandoahVerify) {
1205 verifier()->verify_generic(vo);
1206 } else {
1207 // TODO: Consider allocating verification bitmaps on demand,
1208 // and turn this on unconditionally.
1209 }
1210 }
1211 }
1212 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1213 return _free_set->capacity();
1214 }
1215
1216 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1217 private:
1218 MarkBitMap* _bitmap;
1219 Stack<oop,mtGC>* _oop_stack;
1220
1221 template <class T>
1222 void do_oop_work(T* p) {
1223 T o = RawAccess<>::oop_load(p);
1224 if (!CompressedOops::is_null(o)) {
1225 oop obj = CompressedOops::decode_not_null(o);
1226 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1227 assert(oopDesc::is_oop(obj), "must be a valid oop");
1228 if (!_bitmap->is_marked((HeapWord*) obj)) {
1229 _bitmap->mark((HeapWord*) obj);
1230 _oop_stack->push(obj);
1231 }
1232 }
1233 }
1234 public:
1235 ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1236 _bitmap(bitmap), _oop_stack(oop_stack) {}
1237 void do_oop(oop* p) { do_oop_work(p); }
1238 void do_oop(narrowOop* p) { do_oop_work(p); }
1239 };
1240
1241 /*
1242 * This is public API, used in preparation of object_iterate().
1243 * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1244 * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1245 * control, we call SH::make_tlabs_parsable().
1246 */
1247 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1248 // No-op.
1249 }
1250
1251 /*
1252 * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1253 *
1254 * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1255 * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1256 * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1257 * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1258 * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1259 * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1260 * wiped the bitmap in preparation for next marking).
1261 *
1262 * For all those reasons, we implement object iteration as a single marking traversal, reporting
1263 * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1264 * is allowed to report dead objects, but is not required to do so.
1265 */
1266 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1267 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1268 if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1269 log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1270 return;
1271 }
1272
1273 // Reset bitmap
1274 _aux_bit_map.clear();
1275
1276 Stack<oop,mtGC> oop_stack;
1277
1278 // First, we process all GC roots. This populates the work stack with initial objects.
1279 ShenandoahAllRootScanner rp(1, ShenandoahPhaseTimings::_num_phases);
1280 ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1281 rp.roots_do_unchecked(&oops);
1282
1283 // Work through the oop stack to traverse heap.
1284 while (! oop_stack.is_empty()) {
1285 oop obj = oop_stack.pop();
1286 assert(oopDesc::is_oop(obj), "must be a valid oop");
1287 cl->do_object(obj);
1288 obj->oop_iterate(&oops);
1289 }
1290
1291 assert(oop_stack.is_empty(), "should be empty");
1292
1293 if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1294 log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1295 }
1296 }
1297
1298 void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
1299 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1300 object_iterate(cl);
1301 }
1302
1303 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1304 for (size_t i = 0; i < num_regions(); i++) {
1305 ShenandoahHeapRegion* current = get_region(i);
1306 blk->heap_region_do(current);
1307 }
1308 }
1309
1310 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1311 private:
1312 ShenandoahHeap* const _heap;
1313 ShenandoahHeapRegionClosure* const _blk;
1314
1315 DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
1316 volatile size_t _index;
1317 DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
1318
1319 public:
1320 ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1321 AbstractGangTask("Parallel Region Task"),
1322 _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1323
1324 void work(uint worker_id) {
1325 size_t stride = ShenandoahParallelRegionStride;
1326
1327 size_t max = _heap->num_regions();
1328 while (_index < max) {
1329 size_t cur = Atomic::add(stride, &_index) - stride;
1330 size_t start = cur;
1331 size_t end = MIN2(cur + stride, max);
1332 if (start >= max) break;
1333
1334 for (size_t i = cur; i < end; i++) {
1335 ShenandoahHeapRegion* current = _heap->get_region(i);
1336 _blk->heap_region_do(current);
1337 }
1338 }
1339 }
1340 };
1341
1342 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1343 assert(blk->is_thread_safe(), "Only thread-safe closures here");
1344 if (num_regions() > ShenandoahParallelRegionStride) {
1345 ShenandoahParallelHeapRegionTask task(blk);
1346 workers()->run_task(&task);
1347 } else {
1348 heap_region_iterate(blk);
1349 }
1350 }
1351
1352 class ShenandoahClearLivenessClosure : public ShenandoahHeapRegionClosure {
1353 private:
1354 ShenandoahMarkingContext* const _ctx;
1355 public:
1356 ShenandoahClearLivenessClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1357
1358 void heap_region_do(ShenandoahHeapRegion* r) {
1359 if (r->is_active()) {
1360 r->clear_live_data();
1361 _ctx->capture_top_at_mark_start(r);
1362 } else {
1363 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->region_number());
1364 assert(_ctx->top_at_mark_start(r) == r->top(),
1365 "Region " SIZE_FORMAT " should already have correct TAMS", r->region_number());
1366 }
1367 }
1368
1369 bool is_thread_safe() { return true; }
1370 };
1371
1372 void ShenandoahHeap::op_init_mark() {
1373 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1374 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1375
1376 assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1377 assert(!marking_context()->is_complete(), "should not be complete");
1378
1379 if (ShenandoahVerify) {
1380 verifier()->verify_before_concmark();
1381 }
1382
1383 if (VerifyBeforeGC) {
1384 Universe::verify();
1385 }
1386
1387 set_concurrent_mark_in_progress(true);
1388 // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1389 {
1390 ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
1391 make_parsable(true);
1392 }
1393
1394 {
1395 ShenandoahGCPhase phase(ShenandoahPhaseTimings::clear_liveness);
1396 ShenandoahClearLivenessClosure clc;
1397 parallel_heap_region_iterate(&clc);
1398 }
1399
1400 // Make above changes visible to worker threads
1401 OrderAccess::fence();
1402
1403 concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1404
1405 if (UseTLAB) {
1406 ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1407 resize_tlabs();
1408 }
1409
1410 if (ShenandoahPacing) {
1411 pacer()->setup_for_mark();
1412 }
1413 }
1414
1415 void ShenandoahHeap::op_mark() {
1416 concurrent_mark()->mark_from_roots();
1417 }
1418
1419 class ShenandoahCompleteLivenessClosure : public ShenandoahHeapRegionClosure {
1420 private:
1421 ShenandoahMarkingContext* const _ctx;
1422 public:
1423 ShenandoahCompleteLivenessClosure() : _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
1424
1425 void heap_region_do(ShenandoahHeapRegion* r) {
1426 if (r->is_active()) {
1427 HeapWord *tams = _ctx->top_at_mark_start(r);
1428 HeapWord *top = r->top();
1429 if (top > tams) {
1430 r->increase_live_data_alloc_words(pointer_delta(top, tams));
1431 }
1432 } else {
1433 assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->region_number());
1434 assert(_ctx->top_at_mark_start(r) == r->top(),
1435 "Region " SIZE_FORMAT " should have correct TAMS", r->region_number());
1436 }
1437 }
1438
1439 bool is_thread_safe() { return true; }
1440 };
1441
1442 void ShenandoahHeap::op_final_mark() {
1443 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1444
1445 // It is critical that we
1446 // evacuate roots right after finishing marking, so that we don't
1447 // get unmarked objects in the roots.
1448
1449 if (!cancelled_gc()) {
1450 concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1451
1452 if (has_forwarded_objects()) {
1453 // Degen may be caused by failed evacuation of roots
1454 if (is_degenerated_gc_in_progress()) {
1455 concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
1456 } else {
1457 concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::update_roots);
1458 }
1459 }
1460
1461 if (ShenandoahVerify) {
1462 verifier()->verify_roots_no_forwarded();
1463 }
1464
1465 stop_concurrent_marking();
1466
1467 {
1468 ShenandoahGCPhase phase(ShenandoahPhaseTimings::complete_liveness);
1469
1470 // All allocations past TAMS are implicitly live, adjust the region data.
1471 // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1472 ShenandoahCompleteLivenessClosure cl;
1473 parallel_heap_region_iterate(&cl);
1474 }
1475
1476 {
1477 ShenandoahGCPhase prepare_evac(ShenandoahPhaseTimings::prepare_evac);
1478
1479 make_parsable(true);
1480
1481 trash_cset_regions();
1482
1483 {
1484 ShenandoahHeapLocker locker(lock());
1485 _collection_set->clear();
1486 _free_set->clear();
1487
1488 heuristics()->choose_collection_set(_collection_set);
1489
1490 _free_set->rebuild();
1491 }
1492 }
1493
1494 // If collection set has candidates, start evacuation.
1495 // Otherwise, bypass the rest of the cycle.
1496 if (!collection_set()->is_empty()) {
1497 ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1498
1499 if (ShenandoahVerify) {
1500 verifier()->verify_before_evacuation();
1501 }
1502
1503 set_evacuation_in_progress(true);
1504 // From here on, we need to update references.
1505 set_has_forwarded_objects(true);
1506
1507 evacuate_and_update_roots();
1508
1509 if (ShenandoahPacing) {
1510 pacer()->setup_for_evac();
1511 }
1512
1513 if (ShenandoahVerify) {
1514 verifier()->verify_roots_no_forwarded();
1515 verifier()->verify_during_evacuation();
1516 }
1517 } else {
1518 if (ShenandoahVerify) {
1519 verifier()->verify_after_concmark();
1520 }
1521
1522 if (VerifyAfterGC) {
1523 Universe::verify();
1524 }
1525 }
1526
1527 } else {
1528 concurrent_mark()->cancel();
1529 stop_concurrent_marking();
1530
1531 if (process_references()) {
1532 // Abandon reference processing right away: pre-cleaning must have failed.
1533 ReferenceProcessor *rp = ref_processor();
1534 rp->disable_discovery();
1535 rp->abandon_partial_discovery();
1536 rp->verify_no_references_recorded();
1537 }
1538 }
1539 }
1540
1541 void ShenandoahHeap::op_final_evac() {
1542 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1543
1544 set_evacuation_in_progress(false);
1545
1546 retire_and_reset_gclabs();
1547
1548 if (ShenandoahVerify) {
1549 verifier()->verify_after_evacuation();
1550 }
1551
1552 if (VerifyAfterGC) {
1553 Universe::verify();
1554 }
1555 }
1556
1557 void ShenandoahHeap::op_conc_evac() {
1558 ShenandoahEvacuationTask task(this, _collection_set, true);
1559 workers()->run_task(&task);
1560 }
1561
1562 void ShenandoahHeap::op_stw_evac() {
1563 ShenandoahEvacuationTask task(this, _collection_set, false);
1564 workers()->run_task(&task);
1565 }
1566
1567 void ShenandoahHeap::op_updaterefs() {
1568 update_heap_references(true);
1569 }
1570
1571 void ShenandoahHeap::op_cleanup() {
1572 free_set()->recycle_trash();
1573 }
1574
1575 void ShenandoahHeap::op_reset() {
1576 reset_mark_bitmap();
1577 }
1578
1579 void ShenandoahHeap::op_preclean() {
1580 concurrent_mark()->preclean_weak_refs();
1581 }
1582
1583 void ShenandoahHeap::op_init_traversal() {
1584 traversal_gc()->init_traversal_collection();
1585 }
1586
1587 void ShenandoahHeap::op_traversal() {
1588 traversal_gc()->concurrent_traversal_collection();
1589 }
1590
1591 void ShenandoahHeap::op_final_traversal() {
1592 traversal_gc()->final_traversal_collection();
1593 }
1594
1595 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1596 ShenandoahMetricsSnapshot metrics;
1597 metrics.snap_before();
1598
1599 full_gc()->do_it(cause);
1600 if (UseTLAB) {
1601 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1602 resize_all_tlabs();
1603 }
1604
1605 metrics.snap_after();
1606
1607 if (metrics.is_good_progress()) {
1608 _progress_last_gc.set();
1609 } else {
1610 // Nothing to do. Tell the allocation path that we have failed to make
1611 // progress, and it can finally fail.
1612 _progress_last_gc.unset();
1613 }
1614 }
1615
1616 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1617 // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1618 // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1619 // some phase, we have to upgrade the Degenerate GC to Full GC.
1620
1621 clear_cancelled_gc();
1622
1623 ShenandoahMetricsSnapshot metrics;
1624 metrics.snap_before();
1625
1626 switch (point) {
1627 case _degenerated_traversal:
1628 {
1629 // Drop the collection set. Note: this leaves some already forwarded objects
1630 // behind, which may be problematic, see comments for ShenandoahEvacAssist
1631 // workarounds in ShenandoahTraversalHeuristics.
1632
1633 ShenandoahHeapLocker locker(lock());
1634 collection_set()->clear_current_index();
1635 for (size_t i = 0; i < collection_set()->count(); i++) {
1636 ShenandoahHeapRegion* r = collection_set()->next();
1637 r->make_regular_bypass();
1638 }
1639 collection_set()->clear();
1640 }
1641 op_final_traversal();
1642 op_cleanup();
1643 return;
1644
1645 // The cases below form the Duff's-like device: it describes the actual GC cycle,
1646 // but enters it at different points, depending on which concurrent phase had
1647 // degenerated.
1648
1649 case _degenerated_outside_cycle:
1650 // We have degenerated from outside the cycle, which means something is bad with
1651 // the heap, most probably heavy humongous fragmentation, or we are very low on free
1652 // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1653 // we can do the most aggressive degen cycle, which includes processing references and
1654 // class unloading, unless those features are explicitly disabled.
1655 //
1656 // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1657 // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1658 set_process_references(heuristics()->can_process_references());
1659 set_unload_classes(heuristics()->can_unload_classes());
1660
1661 if (heuristics()->can_do_traversal_gc()) {
1662 // Not possible to degenerate from here, upgrade to Full GC right away.
1663 cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1664 op_degenerated_fail();
1665 return;
1666 }
1667
1668 op_reset();
1669
1670 op_init_mark();
1671 if (cancelled_gc()) {
1672 op_degenerated_fail();
1673 return;
1674 }
1675
1676 case _degenerated_mark:
1677 op_final_mark();
1678 if (cancelled_gc()) {
1679 op_degenerated_fail();
1680 return;
1681 }
1682
1683 op_cleanup();
1684
1685 case _degenerated_evac:
1686 // If heuristics thinks we should do the cycle, this flag would be set,
1687 // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1688 if (is_evacuation_in_progress()) {
1689
1690 // Degeneration under oom-evac protocol might have left some objects in
1691 // collection set un-evacuated. Restart evacuation from the beginning to
1692 // capture all objects. For all the objects that are already evacuated,
1693 // it would be a simple check, which is supposed to be fast. This is also
1694 // safe to do even without degeneration, as CSet iterator is at beginning
1695 // in preparation for evacuation anyway.
1696 //
1697 // Before doing that, we need to make sure we never had any cset-pinned
1698 // regions. This may happen if allocation failure happened when evacuating
1699 // the about-to-be-pinned object, oom-evac protocol left the object in
1700 // the collection set, and then the pin reached the cset region. If we continue
1701 // the cycle here, we would trash the cset and alive objects in it. To avoid
1702 // it, we fail degeneration right away and slide into Full GC to recover.
1703
1704 {
1705 collection_set()->clear_current_index();
1706
1707 ShenandoahHeapRegion* r;
1708 while ((r = collection_set()->next()) != NULL) {
1709 if (r->is_pinned()) {
1710 cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1711 op_degenerated_fail();
1712 return;
1713 }
1714 }
1715
1716 collection_set()->clear_current_index();
1717 }
1718
1719 op_stw_evac();
1720 if (cancelled_gc()) {
1721 op_degenerated_fail();
1722 return;
1723 }
1724 }
1725
1726 // If heuristics thinks we should do the cycle, this flag would be set,
1727 // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
1728 if (has_forwarded_objects()) {
1729 op_init_updaterefs();
1730 if (cancelled_gc()) {
1731 op_degenerated_fail();
1732 return;
1733 }
1734 }
1735
1736 case _degenerated_updaterefs:
1737 if (has_forwarded_objects()) {
1738 op_final_updaterefs();
1739 if (cancelled_gc()) {
1740 op_degenerated_fail();
1741 return;
1742 }
1743 }
1744
1745 op_cleanup();
1746 break;
1747
1748 default:
1749 ShouldNotReachHere();
1750 }
1751
1752 if (ShenandoahVerify) {
1753 verifier()->verify_after_degenerated();
1754 }
1755
1756 if (VerifyAfterGC) {
1757 Universe::verify();
1758 }
1759
1760 metrics.snap_after();
1761
1762 // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
1763 // because that probably means the heap is overloaded and/or fragmented.
1764 if (!metrics.is_good_progress()) {
1765 _progress_last_gc.unset();
1766 cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1767 op_degenerated_futile();
1768 } else {
1769 _progress_last_gc.set();
1770 }
1771 }
1772
1773 void ShenandoahHeap::op_degenerated_fail() {
1774 log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
1775 shenandoah_policy()->record_degenerated_upgrade_to_full();
1776 op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1777 }
1778
1779 void ShenandoahHeap::op_degenerated_futile() {
1780 shenandoah_policy()->record_degenerated_upgrade_to_full();
1781 op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1782 }
1783
1784 void ShenandoahHeap::stop_concurrent_marking() {
1785 assert(is_concurrent_mark_in_progress(), "How else could we get here?");
1786 set_concurrent_mark_in_progress(false);
1787 if (!cancelled_gc()) {
1788 // If we needed to update refs, and concurrent marking has been cancelled,
1789 // we need to finish updating references.
1790 set_has_forwarded_objects(false);
1791 mark_complete_marking_context();
1792 }
1793 }
1794
1795 void ShenandoahHeap::force_satb_flush_all_threads() {
1796 if (!is_concurrent_mark_in_progress() && !is_concurrent_traversal_in_progress()) {
1797 // No need to flush SATBs
1798 return;
1799 }
1800
1801 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1802 ShenandoahThreadLocalData::set_force_satb_flush(t, true);
1803 }
1804 // The threads are not "acquiring" their thread-local data, but it does not
1805 // hurt to "release" the updates here anyway.
1806 OrderAccess::fence();
1807 }
1808
1809 void ShenandoahHeap::set_gc_state_all_threads(char state) {
1810 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1811 ShenandoahThreadLocalData::set_gc_state(t, state);
1812 }
1813 }
1814
1815 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
1816 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
1817 _gc_state.set_cond(mask, value);
1818 set_gc_state_all_threads(_gc_state.raw_value());
1819 }
1820
1821 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1822 if (has_forwarded_objects()) {
1823 set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
1824 } else {
1825 set_gc_state_mask(MARKING, in_progress);
1826 }
1827 ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1828 }
1829
1830 void ShenandoahHeap::set_concurrent_traversal_in_progress(bool in_progress) {
1831 set_gc_state_mask(TRAVERSAL | HAS_FORWARDED | UPDATEREFS, in_progress);
1832 ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1833 }
1834
1835 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1836 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1837 set_gc_state_mask(EVACUATION, in_progress);
1838 }
1839
1840 void ShenandoahHeap::ref_processing_init() {
1841 assert(_max_workers > 0, "Sanity");
1842
1843 _ref_processor =
1844 new ReferenceProcessor(&_subject_to_discovery, // is_subject_to_discovery
1845 ParallelRefProcEnabled, // MT processing
1846 _max_workers, // Degree of MT processing
1847 true, // MT discovery
1848 _max_workers, // Degree of MT discovery
1849 false, // Reference discovery is not atomic
1850 NULL, // No closure, should be installed before use
1851 true); // Scale worker threads
1852
1853 shenandoah_assert_rp_isalive_not_installed();
1854 }
1855
1856 GCTracer* ShenandoahHeap::tracer() {
1857 return shenandoah_policy()->tracer();
1858 }
1859
1860 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1861 return _free_set->used();
1862 }
1863
1864 bool ShenandoahHeap::try_cancel_gc() {
1865 while (true) {
1866 jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1867 if (prev == CANCELLABLE) return true;
1868 else if (prev == CANCELLED) return false;
1869 assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
1870 assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
1871 {
1872 // We need to provide a safepoint here, otherwise we might
1873 // spin forever if a SP is pending.
1874 ThreadBlockInVM sp(JavaThread::current());
1875 SpinPause();
1876 }
1877 }
1878 }
1879
1880 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
1881 if (try_cancel_gc()) {
1882 FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
1883 log_info(gc)("%s", msg.buffer());
1884 Events::log(Thread::current(), "%s", msg.buffer());
1885 }
1886 }
1887
1888 uint ShenandoahHeap::max_workers() {
1889 return _max_workers;
1890 }
1891
1892 void ShenandoahHeap::stop() {
1893 // The shutdown sequence should be able to terminate when GC is running.
1894
1895 // Step 0. Notify policy to disable event recording.
1896 _shenandoah_policy->record_shutdown();
1897
1898 // Step 1. Notify control thread that we are in shutdown.
1899 // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
1900 // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
1901 control_thread()->prepare_for_graceful_shutdown();
1902
1903 // Step 2. Notify GC workers that we are cancelling GC.
1904 cancel_gc(GCCause::_shenandoah_stop_vm);
1905
1906 // Step 3. Wait until GC worker exits normally.
1907 control_thread()->stop();
1908
1909 // Step 4. Stop String Dedup thread if it is active
1910 if (ShenandoahStringDedup::is_enabled()) {
1911 ShenandoahStringDedup::stop();
1912 }
1913 }
1914
1915 void ShenandoahHeap::unload_classes_and_cleanup_tables(bool full_gc) {
1916 assert(heuristics()->can_unload_classes(), "Class unloading should be enabled");
1917
1918 ShenandoahGCPhase root_phase(full_gc ?
1919 ShenandoahPhaseTimings::full_gc_purge :
1920 ShenandoahPhaseTimings::purge);
1921
1922 ShenandoahIsAliveSelector alive;
1923 BoolObjectClosure* is_alive = alive.is_alive_closure();
1924
1925 bool purged_class;
1926
1927 // Unload classes and purge SystemDictionary.
1928 {
1929 ShenandoahGCPhase phase(full_gc ?
1930 ShenandoahPhaseTimings::full_gc_purge_class_unload :
1931 ShenandoahPhaseTimings::purge_class_unload);
1932 purged_class = SystemDictionary::do_unloading(gc_timer());
1933 }
1934
1935 {
1936 ShenandoahGCPhase phase(full_gc ?
1937 ShenandoahPhaseTimings::full_gc_purge_par :
1938 ShenandoahPhaseTimings::purge_par);
1939 uint active = _workers->active_workers();
1940 ParallelCleaningTask unlink_task(is_alive, active, purged_class, true);
1941 _workers->run_task(&unlink_task);
1942 }
1943
1944 {
1945 ShenandoahGCPhase phase(full_gc ?
1946 ShenandoahPhaseTimings::full_gc_purge_cldg :
1947 ShenandoahPhaseTimings::purge_cldg);
1948 ClassLoaderDataGraph::purge();
1949 }
1950 }
1951
1952 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
1953 set_gc_state_mask(HAS_FORWARDED, cond);
1954 }
1955
1956 void ShenandoahHeap::set_process_references(bool pr) {
1957 _process_references.set_cond(pr);
1958 }
1959
1960 void ShenandoahHeap::set_unload_classes(bool uc) {
1961 _unload_classes.set_cond(uc);
1962 }
1963
1964 bool ShenandoahHeap::process_references() const {
1965 return _process_references.is_set();
1966 }
1967
1968 bool ShenandoahHeap::unload_classes() const {
1969 return _unload_classes.is_set();
1970 }
1971
1972 address ShenandoahHeap::in_cset_fast_test_addr() {
1973 ShenandoahHeap* heap = ShenandoahHeap::heap();
1974 assert(heap->collection_set() != NULL, "Sanity");
1975 return (address) heap->collection_set()->biased_map_address();
1976 }
1977
1978 address ShenandoahHeap::cancelled_gc_addr() {
1979 return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
1980 }
1981
1982 address ShenandoahHeap::gc_state_addr() {
1983 return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
1984 }
1985
1986 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
1987 return OrderAccess::load_acquire(&_bytes_allocated_since_gc_start);
1988 }
1989
1990 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
1991 OrderAccess::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
1992 }
1993
1994 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
1995 _degenerated_gc_in_progress.set_cond(in_progress);
1996 }
1997
1998 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
1999 _full_gc_in_progress.set_cond(in_progress);
2000 }
2001
2002 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2003 assert (is_full_gc_in_progress(), "should be");
2004 _full_gc_move_in_progress.set_cond(in_progress);
2005 }
2006
2007 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2008 set_gc_state_mask(UPDATEREFS, in_progress);
2009 }
2010
2011 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2012 ShenandoahCodeRoots::add_nmethod(nm);
2013 }
2014
2015 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2016 ShenandoahCodeRoots::remove_nmethod(nm);
2017 }
2018
2019 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2020 ShenandoahHeapLocker locker(lock());
2021 heap_region_containing(o)->make_pinned();
2022 return o;
2023 }
2024
2025 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2026 ShenandoahHeapLocker locker(lock());
2027 heap_region_containing(o)->make_unpinned();
2028 }
2029
2030 GCTimer* ShenandoahHeap::gc_timer() const {
2031 return _gc_timer;
2032 }
2033
2034 #ifdef ASSERT
2035 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2036 assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2037
2038 if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2039 if (UseDynamicNumberOfGCThreads ||
2040 (FLAG_IS_DEFAULT(ParallelGCThreads) && ForceDynamicNumberOfGCThreads)) {
2041 assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2042 } else {
2043 // Use ParallelGCThreads inside safepoints
2044 assert(nworkers == ParallelGCThreads, "Use ParalleGCThreads within safepoints");
2045 }
2046 } else {
2047 if (UseDynamicNumberOfGCThreads ||
2048 (FLAG_IS_DEFAULT(ConcGCThreads) && ForceDynamicNumberOfGCThreads)) {
2049 assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2050 } else {
2051 // Use ConcGCThreads outside safepoints
2052 assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2053 }
2054 }
2055 }
2056 #endif
2057
2058 ShenandoahVerifier* ShenandoahHeap::verifier() {
2059 guarantee(ShenandoahVerify, "Should be enabled");
2060 assert (_verifier != NULL, "sanity");
2061 return _verifier;
2062 }
2063
2064 template<class T>
2065 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2066 private:
2067 T cl;
2068 ShenandoahHeap* _heap;
2069 ShenandoahRegionIterator* _regions;
2070 bool _concurrent;
2071 public:
2072 ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2073 AbstractGangTask("Concurrent Update References Task"),
2074 cl(T()),
2075 _heap(ShenandoahHeap::heap()),
2076 _regions(regions),
2077 _concurrent(concurrent) {
2078 }
2079
2080 void work(uint worker_id) {
2081 if (_concurrent) {
2082 ShenandoahConcurrentWorkerSession worker_session(worker_id);
2083 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2084 do_work();
2085 } else {
2086 ShenandoahParallelWorkerSession worker_session(worker_id);
2087 do_work();
2088 }
2089 }
2090
2091 private:
2092 void do_work() {
2093 ShenandoahHeapRegion* r = _regions->next();
2094 ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2095 while (r != NULL) {
2096 HeapWord* top_at_start_ur = r->concurrent_iteration_safe_limit();
2097 assert (top_at_start_ur >= r->bottom(), "sanity");
2098 if (r->is_active() && !r->is_cset()) {
2099 _heap->marked_object_oop_iterate(r, &cl, top_at_start_ur);
2100 }
2101 if (ShenandoahPacing) {
2102 _heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
2103 }
2104 if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2105 return;
2106 }
2107 r = _regions->next();
2108 }
2109 }
2110 };
2111
2112 void ShenandoahHeap::update_heap_references(bool concurrent) {
2113 ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2114 workers()->run_task(&task);
2115 }
2116
2117 void ShenandoahHeap::op_init_updaterefs() {
2118 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2119
2120 set_evacuation_in_progress(false);
2121
2122 retire_and_reset_gclabs();
2123
2124 if (ShenandoahVerify) {
2125 if (!is_degenerated_gc_in_progress()) {
2126 verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::ThreadRoots);
2127 }
2128 verifier()->verify_before_updaterefs();
2129 }
2130
2131 set_update_refs_in_progress(true);
2132 make_parsable(true);
2133 for (uint i = 0; i < num_regions(); i++) {
2134 ShenandoahHeapRegion* r = get_region(i);
2135 r->set_concurrent_iteration_safe_limit(r->top());
2136 }
2137
2138 // Reset iterator.
2139 _update_refs_iterator.reset();
2140
2141 if (ShenandoahPacing) {
2142 pacer()->setup_for_updaterefs();
2143 }
2144 }
2145
2146 void ShenandoahHeap::op_final_updaterefs() {
2147 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2148
2149 // Check if there is left-over work, and finish it
2150 if (_update_refs_iterator.has_next()) {
2151 ShenandoahGCPhase final_work(ShenandoahPhaseTimings::final_update_refs_finish_work);
2152
2153 // Finish updating references where we left off.
2154 clear_cancelled_gc();
2155 update_heap_references(false);
2156 }
2157
2158 // Clear cancelled GC, if set. On cancellation path, the block before would handle
2159 // everything. On degenerated paths, cancelled gc would not be set anyway.
2160 if (cancelled_gc()) {
2161 clear_cancelled_gc();
2162 }
2163 assert(!cancelled_gc(), "Should have been done right before");
2164
2165 if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2166 verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::ThreadRoots);
2167 }
2168
2169 if (is_degenerated_gc_in_progress()) {
2170 concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2171 } else {
2172 concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2173 }
2174
2175 ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle);
2176
2177 trash_cset_regions();
2178 set_has_forwarded_objects(false);
2179 set_update_refs_in_progress(false);
2180
2181 if (ShenandoahVerify) {
2182 verifier()->verify_roots_no_forwarded();
2183 verifier()->verify_after_updaterefs();
2184 }
2185
2186 if (VerifyAfterGC) {
2187 Universe::verify();
2188 }
2189
2190 {
2191 ShenandoahHeapLocker locker(lock());
2192 _free_set->rebuild();
2193 }
2194 }
2195
2196 #ifdef ASSERT
2197 void ShenandoahHeap::assert_heaplock_owned_by_current_thread() {
2198 _lock.assert_owned_by_current_thread();
2199 }
2200
2201 void ShenandoahHeap::assert_heaplock_not_owned_by_current_thread() {
2202 _lock.assert_not_owned_by_current_thread();
2203 }
2204
2205 void ShenandoahHeap::assert_heaplock_or_safepoint() {
2206 _lock.assert_owned_by_current_thread_or_safepoint();
2207 }
2208 #endif
2209
2210 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2211 print_on(st);
2212 print_heap_regions_on(st);
2213 }
2214
2215 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2216 size_t slice = r->region_number() / _bitmap_regions_per_slice;
2217
2218 size_t regions_from = _bitmap_regions_per_slice * slice;
2219 size_t regions_to = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2220 for (size_t g = regions_from; g < regions_to; g++) {
2221 assert (g / _bitmap_regions_per_slice == slice, "same slice");
2222 if (skip_self && g == r->region_number()) continue;
2223 if (get_region(g)->is_committed()) {
2224 return true;
2225 }
2226 }
2227 return false;
2228 }
2229
2230 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2231 assert_heaplock_owned_by_current_thread();
2232
2233 // Bitmaps in special regions do not need commits
2234 if (_bitmap_region_special) {
2235 return true;
2236 }
2237
2238 if (is_bitmap_slice_committed(r, true)) {
2239 // Some other region from the group is already committed, meaning the bitmap
2240 // slice is already committed, we exit right away.
2241 return true;
2242 }
2243
2244 // Commit the bitmap slice:
2245 size_t slice = r->region_number() / _bitmap_regions_per_slice;
2246 size_t off = _bitmap_bytes_per_slice * slice;
2247 size_t len = _bitmap_bytes_per_slice;
2248 if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
2249 return false;
2250 }
2251 return true;
2252 }
2253
2254 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2255 assert_heaplock_owned_by_current_thread();
2256
2257 // Bitmaps in special regions do not need uncommits
2258 if (_bitmap_region_special) {
2259 return true;
2260 }
2261
2262 if (is_bitmap_slice_committed(r, true)) {
2263 // Some other region from the group is still committed, meaning the bitmap
2264 // slice is should stay committed, exit right away.
2265 return true;
2266 }
2267
2268 // Uncommit the bitmap slice:
2269 size_t slice = r->region_number() / _bitmap_regions_per_slice;
2270 size_t off = _bitmap_bytes_per_slice * slice;
2271 size_t len = _bitmap_bytes_per_slice;
2272 if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2273 return false;
2274 }
2275 return true;
2276 }
2277
2278 void ShenandoahHeap::safepoint_synchronize_begin() {
2279 if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2280 SuspendibleThreadSet::synchronize();
2281 }
2282 }
2283
2284 void ShenandoahHeap::safepoint_synchronize_end() {
2285 if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2286 SuspendibleThreadSet::desynchronize();
2287 }
2288 }
2289
2290 void ShenandoahHeap::vmop_entry_init_mark() {
2291 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2292 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2293 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark_gross);
2294
2295 try_inject_alloc_failure();
2296 VM_ShenandoahInitMark op;
2297 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2298 }
2299
2300 void ShenandoahHeap::vmop_entry_final_mark() {
2301 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2302 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2303 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark_gross);
2304
2305 try_inject_alloc_failure();
2306 VM_ShenandoahFinalMarkStartEvac op;
2307 VMThread::execute(&op); // jump to entry_final_mark under safepoint
2308 }
2309
2310 void ShenandoahHeap::vmop_entry_final_evac() {
2311 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2312 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2313 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_gross);
2314
2315 VM_ShenandoahFinalEvac op;
2316 VMThread::execute(&op); // jump to entry_final_evac under safepoint
2317 }
2318
2319 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2320 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2321 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2322 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_gross);
2323
2324 try_inject_alloc_failure();
2325 VM_ShenandoahInitUpdateRefs op;
2326 VMThread::execute(&op);
2327 }
2328
2329 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2330 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2331 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2332 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_gross);
2333
2334 try_inject_alloc_failure();
2335 VM_ShenandoahFinalUpdateRefs op;
2336 VMThread::execute(&op);
2337 }
2338
2339 void ShenandoahHeap::vmop_entry_init_traversal() {
2340 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2341 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2342 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc_gross);
2343
2344 try_inject_alloc_failure();
2345 VM_ShenandoahInitTraversalGC op;
2346 VMThread::execute(&op);
2347 }
2348
2349 void ShenandoahHeap::vmop_entry_final_traversal() {
2350 TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2351 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2352 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc_gross);
2353
2354 try_inject_alloc_failure();
2355 VM_ShenandoahFinalTraversalGC op;
2356 VMThread::execute(&op);
2357 }
2358
2359 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2360 TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2361 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2362 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_gross);
2363
2364 try_inject_alloc_failure();
2365 VM_ShenandoahFullGC op(cause);
2366 VMThread::execute(&op);
2367 }
2368
2369 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2370 TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2371 ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2372 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_gross);
2373
2374 VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2375 VMThread::execute(°enerated_gc);
2376 }
2377
2378 void ShenandoahHeap::entry_init_mark() {
2379 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2380 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark);
2381 const char* msg = init_mark_event_message();
2382 GCTraceTime(Info, gc) time(msg, gc_timer());
2383 EventMark em("%s", msg);
2384
2385 ShenandoahWorkerScope scope(workers(),
2386 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2387 "init marking");
2388
2389 op_init_mark();
2390 }
2391
2392 void ShenandoahHeap::entry_final_mark() {
2393 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2394 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark);
2395 const char* msg = final_mark_event_message();
2396 GCTraceTime(Info, gc) time(msg, gc_timer());
2397 EventMark em("%s", msg);
2398
2399 ShenandoahWorkerScope scope(workers(),
2400 ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2401 "final marking");
2402
2403 op_final_mark();
2404 }
2405
2406 void ShenandoahHeap::entry_final_evac() {
2407 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2408 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac);
2409 static const char* msg = "Pause Final Evac";
2410 GCTraceTime(Info, gc) time(msg, gc_timer());
2411 EventMark em("%s", msg);
2412
2413 op_final_evac();
2414 }
2415
2416 void ShenandoahHeap::entry_init_updaterefs() {
2417 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2418 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs);
2419
2420 static const char* msg = "Pause Init Update Refs";
2421 GCTraceTime(Info, gc) time(msg, gc_timer());
2422 EventMark em("%s", msg);
2423
2424 // No workers used in this phase, no setup required
2425
2426 op_init_updaterefs();
2427 }
2428
2429 void ShenandoahHeap::entry_final_updaterefs() {
2430 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2431 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs);
2432
2433 static const char* msg = "Pause Final Update Refs";
2434 GCTraceTime(Info, gc) time(msg, gc_timer());
2435 EventMark em("%s", msg);
2436
2437 ShenandoahWorkerScope scope(workers(),
2438 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2439 "final reference update");
2440
2441 op_final_updaterefs();
2442 }
2443
2444 void ShenandoahHeap::entry_init_traversal() {
2445 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2446 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc);
2447
2448 static const char* msg = "Pause Init Traversal";
2449 GCTraceTime(Info, gc) time(msg, gc_timer());
2450 EventMark em("%s", msg);
2451
2452 ShenandoahWorkerScope scope(workers(),
2453 ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(),
2454 "init traversal");
2455
2456 op_init_traversal();
2457 }
2458
2459 void ShenandoahHeap::entry_final_traversal() {
2460 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2461 ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc);
2462
2463 static const char* msg = "Pause Final Traversal";
2464 GCTraceTime(Info, gc) time(msg, gc_timer());
2465 EventMark em("%s", msg);
2466
2467 ShenandoahWorkerScope scope(workers(),
2468 ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(),
2469 "final traversal");
2470
2471 op_final_traversal();
2472 }
2473
2474 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2475 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2476 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc);
2477
2478 static const char* msg = "Pause Full";
2479 GCTraceTime(Info, gc) time(msg, gc_timer(), cause, true);
2480 EventMark em("%s", msg);
2481
2482 ShenandoahWorkerScope scope(workers(),
2483 ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2484 "full gc");
2485
2486 op_full(cause);
2487 }
2488
2489 void ShenandoahHeap::entry_degenerated(int point) {
2490 ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2491 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc);
2492
2493 ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2494 const char* msg = degen_event_message(dpoint);
2495 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2496 EventMark em("%s", msg);
2497
2498 ShenandoahWorkerScope scope(workers(),
2499 ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2500 "stw degenerated gc");
2501
2502 set_degenerated_gc_in_progress(true);
2503 op_degenerated(dpoint);
2504 set_degenerated_gc_in_progress(false);
2505 }
2506
2507 void ShenandoahHeap::entry_mark() {
2508 TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2509
2510 const char* msg = conc_mark_event_message();
2511 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2512 EventMark em("%s", msg);
2513
2514 ShenandoahWorkerScope scope(workers(),
2515 ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2516 "concurrent marking");
2517
2518 try_inject_alloc_failure();
2519 op_mark();
2520 }
2521
2522 void ShenandoahHeap::entry_evac() {
2523 ShenandoahGCPhase conc_evac_phase(ShenandoahPhaseTimings::conc_evac);
2524 TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2525
2526 static const char* msg = "Concurrent evacuation";
2527 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2528 EventMark em("%s", msg);
2529
2530 ShenandoahWorkerScope scope(workers(),
2531 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2532 "concurrent evacuation");
2533
2534 try_inject_alloc_failure();
2535 op_conc_evac();
2536 }
2537
2538 void ShenandoahHeap::entry_updaterefs() {
2539 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2540
2541 static const char* msg = "Concurrent update references";
2542 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2543 EventMark em("%s", msg);
2544
2545 ShenandoahWorkerScope scope(workers(),
2546 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2547 "concurrent reference update");
2548
2549 try_inject_alloc_failure();
2550 op_updaterefs();
2551 }
2552 void ShenandoahHeap::entry_cleanup() {
2553 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2554
2555 static const char* msg = "Concurrent cleanup";
2556 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2557 EventMark em("%s", msg);
2558
2559 // This phase does not use workers, no need for setup
2560
2561 try_inject_alloc_failure();
2562 op_cleanup();
2563 }
2564
2565 void ShenandoahHeap::entry_reset() {
2566 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2567
2568 static const char* msg = "Concurrent reset";
2569 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2570 EventMark em("%s", msg);
2571
2572 ShenandoahWorkerScope scope(workers(),
2573 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2574 "concurrent reset");
2575
2576 try_inject_alloc_failure();
2577 op_reset();
2578 }
2579
2580 void ShenandoahHeap::entry_preclean() {
2581 if (ShenandoahPreclean && process_references()) {
2582 static const char* msg = "Concurrent precleaning";
2583 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2584 EventMark em("%s", msg);
2585
2586 ShenandoahGCPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
2587
2588 ShenandoahWorkerScope scope(workers(),
2589 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2590 "concurrent preclean",
2591 /* check_workers = */ false);
2592
2593 try_inject_alloc_failure();
2594 op_preclean();
2595 }
2596 }
2597
2598 void ShenandoahHeap::entry_traversal() {
2599 static const char* msg = "Concurrent traversal";
2600 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2601 EventMark em("%s", msg);
2602
2603 TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2604
2605 ShenandoahWorkerScope scope(workers(),
2606 ShenandoahWorkerPolicy::calc_workers_for_conc_traversal(),
2607 "concurrent traversal");
2608
2609 try_inject_alloc_failure();
2610 op_traversal();
2611 }
2612
2613 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2614 static const char *msg = "Concurrent uncommit";
2615 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2616 EventMark em("%s", msg);
2617
2618 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_uncommit);
2619
2620 op_uncommit(shrink_before);
2621 }
2622
2623 void ShenandoahHeap::try_inject_alloc_failure() {
2624 if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2625 _inject_alloc_failure.set();
2626 os::naked_short_sleep(1);
2627 if (cancelled_gc()) {
2628 log_info(gc)("Allocation failure was successfully injected");
2629 }
2630 }
2631 }
2632
2633 bool ShenandoahHeap::should_inject_alloc_failure() {
2634 return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2635 }
2636
2637 void ShenandoahHeap::initialize_serviceability() {
2638 _memory_pool = new ShenandoahMemoryPool(this);
2639 _cycle_memory_manager.add_pool(_memory_pool);
2640 _stw_memory_manager.add_pool(_memory_pool);
2641 }
2642
2643 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2644 GrowableArray<GCMemoryManager*> memory_managers(2);
2645 memory_managers.append(&_cycle_memory_manager);
2646 memory_managers.append(&_stw_memory_manager);
2647 return memory_managers;
2648 }
2649
2650 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2651 GrowableArray<MemoryPool*> memory_pools(1);
2652 memory_pools.append(_memory_pool);
2653 return memory_pools;
2654 }
2655
2656 MemoryUsage ShenandoahHeap::memory_usage() {
2657 return _memory_pool->get_memory_usage();
2658 }
2659
2660 void ShenandoahHeap::enter_evacuation() {
2661 _oom_evac_handler.enter_evacuation();
2662 }
2663
2664 void ShenandoahHeap::leave_evacuation() {
2665 _oom_evac_handler.leave_evacuation();
2666 }
2667
2668 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2669 _heap(ShenandoahHeap::heap()),
2670 _index(0) {}
2671
2672 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2673 _heap(heap),
2674 _index(0) {}
2675
2676 void ShenandoahRegionIterator::reset() {
2677 _index = 0;
2678 }
2679
2680 bool ShenandoahRegionIterator::has_next() const {
2681 return _index < _heap->num_regions();
2682 }
2683
2684 char ShenandoahHeap::gc_state() const {
2685 return _gc_state.raw_value();
2686 }
2687
2688 void ShenandoahHeap::deduplicate_string(oop str) {
2689 assert(java_lang_String::is_instance(str), "invariant");
2690
2691 if (ShenandoahStringDedup::is_enabled()) {
2692 ShenandoahStringDedup::deduplicate(str);
2693 }
2694 }
2695
2696 const char* ShenandoahHeap::init_mark_event_message() const {
2697 bool update_refs = has_forwarded_objects();
2698 bool proc_refs = process_references();
2699 bool unload_cls = unload_classes();
2700
2701 if (update_refs && proc_refs && unload_cls) {
2702 return "Pause Init Mark (update refs) (process weakrefs) (unload classes)";
2703 } else if (update_refs && proc_refs) {
2704 return "Pause Init Mark (update refs) (process weakrefs)";
2705 } else if (update_refs && unload_cls) {
2706 return "Pause Init Mark (update refs) (unload classes)";
2707 } else if (proc_refs && unload_cls) {
2708 return "Pause Init Mark (process weakrefs) (unload classes)";
2709 } else if (update_refs) {
2710 return "Pause Init Mark (update refs)";
2711 } else if (proc_refs) {
2712 return "Pause Init Mark (process weakrefs)";
2713 } else if (unload_cls) {
2714 return "Pause Init Mark (unload classes)";
2715 } else {
2716 return "Pause Init Mark";
2717 }
2718 }
2719
2720 const char* ShenandoahHeap::final_mark_event_message() const {
2721 bool update_refs = has_forwarded_objects();
2722 bool proc_refs = process_references();
2723 bool unload_cls = unload_classes();
2724
2725 if (update_refs && proc_refs && unload_cls) {
2726 return "Pause Final Mark (update refs) (process weakrefs) (unload classes)";
2727 } else if (update_refs && proc_refs) {
2728 return "Pause Final Mark (update refs) (process weakrefs)";
2729 } else if (update_refs && unload_cls) {
2730 return "Pause Final Mark (update refs) (unload classes)";
2731 } else if (proc_refs && unload_cls) {
2732 return "Pause Final Mark (process weakrefs) (unload classes)";
2733 } else if (update_refs) {
2734 return "Pause Final Mark (update refs)";
2735 } else if (proc_refs) {
2736 return "Pause Final Mark (process weakrefs)";
2737 } else if (unload_cls) {
2738 return "Pause Final Mark (unload classes)";
2739 } else {
2740 return "Pause Final Mark";
2741 }
2742 }
2743
2744 const char* ShenandoahHeap::conc_mark_event_message() const {
2745 bool update_refs = has_forwarded_objects();
2746 bool proc_refs = process_references();
2747 bool unload_cls = unload_classes();
2748
2749 if (update_refs && proc_refs && unload_cls) {
2750 return "Concurrent marking (update refs) (process weakrefs) (unload classes)";
2751 } else if (update_refs && proc_refs) {
2752 return "Concurrent marking (update refs) (process weakrefs)";
2753 } else if (update_refs && unload_cls) {
2754 return "Concurrent marking (update refs) (unload classes)";
2755 } else if (proc_refs && unload_cls) {
2756 return "Concurrent marking (process weakrefs) (unload classes)";
2757 } else if (update_refs) {
2758 return "Concurrent marking (update refs)";
2759 } else if (proc_refs) {
2760 return "Concurrent marking (process weakrefs)";
2761 } else if (unload_cls) {
2762 return "Concurrent marking (unload classes)";
2763 } else {
2764 return "Concurrent marking";
2765 }
2766 }
2767
2768 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
2769 switch (point) {
2770 case _degenerated_unset:
2771 return "Pause Degenerated GC (<UNSET>)";
2772 case _degenerated_traversal:
2773 return "Pause Degenerated GC (Traversal)";
2774 case _degenerated_outside_cycle:
2775 return "Pause Degenerated GC (Outside of Cycle)";
2776 case _degenerated_mark:
2777 return "Pause Degenerated GC (Mark)";
2778 case _degenerated_evac:
2779 return "Pause Degenerated GC (Evacuation)";
2780 case _degenerated_updaterefs:
2781 return "Pause Degenerated GC (Update Refs)";
2782 default:
2783 ShouldNotReachHere();
2784 return "ERROR";
2785 }
2786 }
2787
2788 jushort* ShenandoahHeap::get_liveness_cache(uint worker_id) {
2789 #ifdef ASSERT
2790 assert(_liveness_cache != NULL, "sanity");
2791 assert(worker_id < _max_workers, "sanity");
2792 for (uint i = 0; i < num_regions(); i++) {
2793 assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
2794 }
2795 #endif
2796 return _liveness_cache[worker_id];
2797 }
2798
2799 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2800 assert(worker_id < _max_workers, "sanity");
2801 assert(_liveness_cache != NULL, "sanity");
2802 jushort* ld = _liveness_cache[worker_id];
2803 for (uint i = 0; i < num_regions(); i++) {
2804 ShenandoahHeapRegion* r = get_region(i);
2805 jushort live = ld[i];
2806 if (live > 0) {
2807 r->increase_live_data_gc_words(live);
2808 ld[i] = 0;
2809 }
2810 }
2811 }