1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc_implementation/shared/gcTimer.hpp"
26 #include "gc_implementation/shenandoah/shenandoahGCTraceTime.hpp"
27 #include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp"
28 #include "gc_implementation/shenandoah/shenandoahControlThread.hpp"
29 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
30 #include "gc_implementation/shenandoah/shenandoahFreeSet.hpp"
31 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc_implementation/shenandoah/shenandoahHeuristics.hpp"
33 #include "gc_implementation/shenandoah/shenandoahMonitoringSupport.hpp"
34 #include "gc_implementation/shenandoah/shenandoahPhaseTimings.hpp"
35 #include "gc_implementation/shenandoah/shenandoahUtils.hpp"
36 #include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp"
37 #include "gc_implementation/shenandoah/vm_operations_shenandoah.hpp"
38 #include "memory/iterator.hpp"
39 #include "memory/universe.hpp"
40 #include "runtime/vmThread.hpp"
41
42 #ifdef _WINDOWS
43 #pragma warning(disable : 4355)
44 #endif
45
46 SurrogateLockerThread* ShenandoahControlThread::_slt = NULL;
47
48 ShenandoahControlThread::ShenandoahControlThread() :
49 ConcurrentGCThread(),
50 _alloc_failure_waiters_lock(Mutex::leaf, "ShenandoahAllocFailureGC_lock", true),
51 _explicit_gc_waiters_lock(Mutex::leaf, "ShenandoahExplicitGC_lock", true),
52 _periodic_task(this),
53 _allocs_seen(0),
54 _explicit_gc_cause(GCCause::_no_cause_specified),
55 _degen_point(ShenandoahHeap::_degenerated_outside_cycle) {
56
57 create_and_start();
58 _periodic_task.enroll();
59 _periodic_satb_flush_task.enroll();
60 }
61
62 ShenandoahControlThread::~ShenandoahControlThread() {
63 // This is here so that super is called.
64 }
65
66 void ShenandoahPeriodicTask::task() {
67 _thread->handle_force_counters_update();
68 _thread->handle_counters_update();
69 }
70
71 void ShenandoahPeriodicSATBFlushTask::task() {
72 ShenandoahHeap::heap()->force_satb_flush_all_threads();
73 }
74
75 void ShenandoahControlThread::run() {
76 initialize_in_thread();
77
78 wait_for_universe_init();
79
80 // Wait until we have the surrogate locker thread in place.
81 {
82 MutexLockerEx x(CGC_lock, true);
83 while(_slt == NULL && !_should_terminate) {
84 CGC_lock->wait(true, 200);
85 }
86 }
87
88 ShenandoahHeap* heap = ShenandoahHeap::heap();
89
90 int sleep = ShenandoahControlIntervalMin;
91
92 double last_shrink_time = os::elapsedTime();
93 double last_sleep_adjust_time = os::elapsedTime();
94
95 // Shrink period avoids constantly polling regions for shrinking.
96 // Having a period 10x lower than the delay would mean we hit the
97 // shrinking with lag of less than 1/10-th of true delay.
98 // ShenandoahUncommitDelay is in msecs, but shrink_period is in seconds.
99 double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10;
100
101 ShenandoahCollectorPolicy* policy = heap->shenandoahPolicy();
102
103 ShenandoahHeuristics* heuristics = heap->heuristics();
104 while (!in_graceful_shutdown() && !_should_terminate) {
105 // Figure out if we have pending requests.
106 bool alloc_failure_pending = _alloc_failure_gc.is_set();
107 bool explicit_gc_requested = _explicit_gc.is_set();
108
109 // This control loop iteration have seen this much allocations.
110 intptr_t allocs_seen = (intptr_t)(Atomic::xchg_ptr(0, &_allocs_seen));
111
112 // Choose which GC mode to run in. The block below should select a single mode.
113 GCMode mode = none;
114 GCCause::Cause cause = GCCause::_last_gc_cause;
115 ShenandoahHeap::ShenandoahDegenPoint degen_point = ShenandoahHeap::_degenerated_unset;
116
117 if (alloc_failure_pending) {
118 // Allocation failure takes precedence: we have to deal with it first thing
119 log_info(gc)("Trigger: Handle Allocation Failure");
120
121 cause = GCCause::_allocation_failure;
122
123 // Consume the degen point, and seed it with default value
124 degen_point = _degen_point;
125 _degen_point = ShenandoahHeap::_degenerated_outside_cycle;
126
127 if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle()) {
128 heuristics->record_allocation_failure_gc();
129 policy->record_alloc_failure_to_degenerated(degen_point);
130 mode = stw_degenerated;
131 } else {
132 heuristics->record_allocation_failure_gc();
133 policy->record_alloc_failure_to_full();
134 mode = stw_full;
135 }
136
137 } else if (explicit_gc_requested) {
138 // Honor explicit GC requests
139 log_info(gc)("Trigger: Explicit GC request");
140
141 cause = _explicit_gc_cause;
142
143 if (ExplicitGCInvokesConcurrent) {
144 heuristics->record_explicit_gc();
145 policy->record_explicit_to_concurrent();
146 mode = concurrent_normal;
147 } else {
148 heuristics->record_explicit_gc();
149 policy->record_explicit_to_full();
150 mode = stw_full;
151 }
152 } else {
153 // Potential normal cycle: ask heuristics if it wants to act
154 if (heuristics->should_start_normal_gc()) {
155 mode = concurrent_normal;
156 cause = GCCause::_shenandoah_concurrent_gc;
157 }
158
159 // Ask policy if this cycle wants to process references or unload classes
160 heap->set_process_references(heuristics->should_process_references());
161 heap->set_unload_classes(heuristics->should_unload_classes());
162 }
163
164 // Blow all soft references on this cycle, if handling allocation failure,
165 // or we are requested to do so unconditionally.
166 if (alloc_failure_pending || ShenandoahAlwaysClearSoftRefs) {
167 heap->collector_policy()->set_should_clear_all_soft_refs(true);
168 }
169
170 bool gc_requested = (mode != none);
171 assert (!gc_requested || cause != GCCause::_last_gc_cause, "GC cause should be set");
172
173 if (gc_requested) {
174 heap->reset_bytes_allocated_since_gc_start();
175
176 // If GC was requested, we are sampling the counters even without actual triggers
177 // from allocation machinery. This captures GC phases more accurately.
178 set_forced_counters_update(true);
179
180 // If GC was requested, we better dump freeset data for performance debugging
181 {
182 ShenandoahHeapLocker locker(heap->lock());
183 heap->free_set()->log_status();
184 }
185 }
186
187 switch (mode) {
188 case none:
189 break;
190 case concurrent_normal:
191 service_concurrent_normal_cycle(cause);
192 break;
193 case stw_degenerated:
194 service_stw_degenerated_cycle(cause, degen_point);
195 break;
196 case stw_full:
197 service_stw_full_cycle(cause);
198 break;
199 default:
200 ShouldNotReachHere();
201 }
202
203 if (gc_requested) {
204 // If this was the explicit GC cycle, notify waiters about it
205 if (explicit_gc_requested) {
206 notify_explicit_gc_waiters();
207 }
208
209 // If this was the allocation failure GC cycle, notify waiters about it
210 if (alloc_failure_pending) {
211 notify_alloc_failure_waiters();
212 }
213
214 // Report current free set state at the end of cycle, whether
215 // it is a normal completion, or the abort.
216 {
217 ShenandoahHeapLocker locker(heap->lock());
218 heap->free_set()->log_status();
219
220 // Notify Universe about new heap usage. This has implications for
221 // global soft refs policy, and we better report it every time heap
222 // usage goes down.
223 Universe::update_heap_info_at_gc();
224 }
225
226 // Disable forced counters update, and update counters one more time
227 // to capture the state at the end of GC session.
228 handle_force_counters_update();
229 set_forced_counters_update(false);
230
231 // Retract forceful part of soft refs policy
232 heap->collector_policy()->set_should_clear_all_soft_refs(false);
233
234 // GC is over, we are at idle now
235 if (ShenandoahPacing) {
236 heap->pacer()->setup_for_idle();
237 }
238 } else {
239 // Allow allocators to know we have seen this much regions
240 if (ShenandoahPacing && (allocs_seen > 0)) {
241 heap->pacer()->report_alloc(allocs_seen);
242 }
243 }
244
245 double current = os::elapsedTime();
246
247 if (ShenandoahUncommit && (explicit_gc_requested || (current - last_shrink_time > shrink_period))) {
248 // Try to uncommit enough stale regions. Explicit GC tries to uncommit everything.
249 // Regular paths uncommit only occasionally.
250 double shrink_before = explicit_gc_requested ?
251 current :
252 current - (ShenandoahUncommitDelay / 1000.0);
253 service_uncommit(shrink_before);
254 last_shrink_time = current;
255 }
256
257 // Wait before performing the next action. If allocation happened during this wait,
258 // we exit sooner, to let heuristics re-evaluate new conditions. If we are at idle,
259 // back off exponentially.
260 if (_heap_changed.try_unset()) {
261 sleep = ShenandoahControlIntervalMin;
262 } else if ((current - last_sleep_adjust_time) * 1000 > ShenandoahControlIntervalAdjustPeriod){
263 sleep = MIN2<int>(ShenandoahControlIntervalMax, MAX2(1, sleep * 2));
264 last_sleep_adjust_time = current;
265 }
266 os::naked_short_sleep(sleep);
267 }
268
269 // Wait for the actual stop(), can't leave run_service() earlier.
270 while (! _should_terminate) {
271 os::naked_short_sleep(ShenandoahControlIntervalMin);
272 }
273 terminate();
274 }
275
276 void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cause) {
277 // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during
278 // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there.
279 // If second allocation failure happens during Degenerated GC cycle (for example, when GC
280 // tries to evac something and no memory is available), cycle degrades to Full GC.
281 //
282 // There are also two shortcuts through the normal cycle: a) immediate garbage shortcut, when
283 // heuristics says there are no regions to compact, and all the collection comes from immediately
284 // reclaimable regions; b) coalesced UR shortcut, when heuristics decides to coalesce UR with the
285 // mark from the next cycle.
286 //
287 // ................................................................................................
288 //
289 // (immediate garbage shortcut) Concurrent GC
290 // /-------------------------------------------\
291 // | (coalesced UR) v
292 // | /----------------------->o
293 // | | |
294 // | | v
295 // [START] ----> Conc Mark ----o----> Conc Evac --o--> Conc Update-Refs ---o----> [END]
296 // | | | ^
297 // | (af) | (af) | (af) |
298 // ..................|....................|.................|..............|.......................
299 // | | | |
300 // | | | | Degenerated GC
301 // v v v |
302 // STW Mark ----------> STW Evac ----> STW Update-Refs ----->o
303 // | | | ^
304 // | (af) | (af) | (af) |
305 // ..................|....................|.................|..............|.......................
306 // | | | |
307 // | v | | Full GC
308 // \------------------->o<----------------/ |
309 // | |
310 // v |
311 // Full GC --------------------------/
312 //
313
314 ShenandoahHeap* heap = ShenandoahHeap::heap();
315
316 if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_outside_cycle)) return;
317
318 ShenandoahGCSession session(cause);
319
320 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
321
322 // Start initial mark under STW
323 heap->vmop_entry_init_mark();
324
325 // Continue concurrent mark
326 heap->entry_mark();
327 if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_mark)) return;
328
329 // If not cancelled, can try to concurrently pre-clean
330 heap->entry_preclean();
331
332 // Complete marking under STW, and start evacuation
333 heap->vmop_entry_final_mark();
334
335 // Continue the cycle with evacuation and optional update-refs.
336 // This may be skipped if there is nothing to evacuate.
337 // If so, evac_in_progress would be unset by collection set preparation code.
338 if (heap->is_evacuation_in_progress()) {
339 // Final mark had reclaimed some immediate garbage, kick cleanup to reclaim the space
340 // for the rest of the cycle, and report current state of free set.
341 heap->entry_cleanup();
342
343 {
344 ShenandoahHeapLocker locker(heap->lock());
345 heap->free_set()->log_status();
346 }
347
348 // Concurrently evacuate
349 heap->entry_evac();
350 if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_evac)) return;
351
352 // Perform update-refs phase, if required. This phase can be skipped if heuristics
353 // decides to piggy-back the update-refs on the next marking cycle. On either path,
354 // we need to turn off evacuation: either in init-update-refs, or in final-evac.
355 if (heap->heuristics()->should_start_update_refs()) {
356 heap->vmop_entry_init_updaterefs();
357 heap->entry_updaterefs();
358 if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_updaterefs)) return;
359
360 heap->vmop_entry_final_updaterefs();
361 } else {
362 heap->vmop_entry_final_evac();
363 }
364 }
365
366 // Reclaim space and prepare for the next normal cycle:
367 heap->entry_cleanup_bitmaps();
368
369 // Cycle is complete
370 heap->heuristics()->record_success_concurrent();
371 heap->shenandoahPolicy()->record_success_concurrent();
372 }
373
374 bool ShenandoahControlThread::check_cancellation_or_degen(ShenandoahHeap::ShenandoahDegenPoint point) {
375 ShenandoahHeap* heap = ShenandoahHeap::heap();
376 if (heap->cancelled_gc()) {
377 assert (is_alloc_failure_gc() || in_graceful_shutdown(), "Cancel GC either for alloc failure GC, or gracefully exiting");
378 if (!in_graceful_shutdown()) {
379 assert (_degen_point == ShenandoahHeap::_degenerated_outside_cycle,
380 err_msg("Should not be set yet: %s", ShenandoahHeap::degen_point_to_string(_degen_point)));
381 _degen_point = point;
382 }
383 return true;
384 }
385 return false;
386 }
387
388 void ShenandoahControlThread::stop() {
389 {
390 MutexLockerEx ml(Terminator_lock);
391 _should_terminate = true;
392 }
393
394 {
395 MutexLockerEx ml(CGC_lock, Mutex::_no_safepoint_check_flag);
396 CGC_lock->notify_all();
397 }
398
399 {
400 MutexLockerEx ml(Terminator_lock);
401 while (!_has_terminated) {
402 Terminator_lock->wait();
403 }
404 }
405 }
406
407 void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) {
408 ShenandoahHeap* heap = ShenandoahHeap::heap();
409 ShenandoahGCSession session(cause);
410
411 heap->vmop_entry_full(cause);
412
413 heap->heuristics()->record_success_full();
414 heap->shenandoahPolicy()->record_success_full();
415 }
416
417 void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahHeap::ShenandoahDegenPoint point) {
418 assert (point != ShenandoahHeap::_degenerated_unset, "Degenerated point should be set");
419 ShenandoahHeap* heap = ShenandoahHeap::heap();
420 ShenandoahGCSession session(cause);
421
422 heap->vmop_degenerated(point);
423
424 heap->heuristics()->record_success_degenerated();
425 heap->shenandoahPolicy()->record_success_degenerated();
426 }
427
428 void ShenandoahControlThread::service_uncommit(double shrink_before) {
429 ShenandoahHeap* heap = ShenandoahHeap::heap();
430
431 // Scan through the heap and determine if there is work to do. This avoids taking
432 // heap lock if there is no work available, avoids spamming logs with superfluous
433 // logging messages, and minimises the amount of work while locks are taken.
434
435 bool has_work = false;
436 for (size_t i = 0; i < heap->num_regions(); i++) {
437 ShenandoahHeapRegion *r = heap->get_region(i);
438 if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
439 has_work = true;
440 break;
441 }
442 }
443
444 if (has_work) {
445 heap->entry_uncommit(shrink_before);
446 }
447 }
448
449 void ShenandoahControlThread::handle_explicit_gc(GCCause::Cause cause) {
450 assert(GCCause::is_user_requested_gc(cause) ||
451 GCCause::is_serviceability_requested_gc(cause) ||
452 cause == GCCause::_full_gc_alot,
453 "only requested GCs here");
454 if (!DisableExplicitGC) {
455 _explicit_gc_cause = cause;
456
457 _explicit_gc.set();
458 MonitorLockerEx ml(&_explicit_gc_waiters_lock);
459 while (_explicit_gc.is_set()) {
460 ml.wait();
461 }
462 }
463 }
464
465 void ShenandoahControlThread::handle_alloc_failure(size_t words) {
466 ShenandoahHeap* heap = ShenandoahHeap::heap();
467
468 assert(current()->is_Java_thread(), "expect Java thread here");
469
470 if (try_set_alloc_failure_gc()) {
471 // Only report the first allocation failure
472 log_info(gc)("Failed to allocate " SIZE_FORMAT "%s",
473 byte_size_in_proper_unit(words * HeapWordSize), proper_unit_for_byte_size(words * HeapWordSize));
474
475 // Now that alloc failure GC is scheduled, we can abort everything else
476 heap->cancel_gc(GCCause::_allocation_failure);
477 }
478
479 MonitorLockerEx ml(&_alloc_failure_waiters_lock);
480 while (is_alloc_failure_gc()) {
481 ml.wait();
482 }
483 }
484
485 void ShenandoahControlThread::handle_alloc_failure_evac(size_t words) {
486 Thread* t = Thread::current();
487
488 ShenandoahHeap* heap = ShenandoahHeap::heap();
489
490 if (try_set_alloc_failure_gc()) {
491 // Only report the first allocation failure
492 log_info(gc)("Failed to allocate " SIZE_FORMAT "%s for evacuation",
493 byte_size_in_proper_unit(words * HeapWordSize), proper_unit_for_byte_size(words * HeapWordSize));
494 }
495
496 heap->cancel_gc(GCCause::_shenandoah_allocation_failure_evac);
497 }
498
499 void ShenandoahControlThread::notify_alloc_failure_waiters() {
500 _alloc_failure_gc.unset();
501 MonitorLockerEx ml(&_alloc_failure_waiters_lock);
502 ml.notify_all();
503 }
504
505 bool ShenandoahControlThread::try_set_alloc_failure_gc() {
506 return _alloc_failure_gc.try_set();
507 }
508
509 bool ShenandoahControlThread::is_alloc_failure_gc() {
510 return _alloc_failure_gc.is_set();
511 }
512
513 void ShenandoahControlThread::notify_explicit_gc_waiters() {
514 _explicit_gc.unset();
515 MonitorLockerEx ml(&_explicit_gc_waiters_lock);
516 ml.notify_all();
517 }
518
519 void ShenandoahControlThread::handle_counters_update() {
520 if (_do_counters_update.is_set()) {
521 _do_counters_update.unset();
522 ShenandoahHeap::heap()->monitoring_support()->update_counters();
523 }
524 }
525
526 void ShenandoahControlThread::handle_force_counters_update() {
527 if (_force_counters_update.is_set()) {
528 _do_counters_update.unset(); // reset these too, we do update now!
529 ShenandoahHeap::heap()->monitoring_support()->update_counters();
530 }
531 }
532
533 void ShenandoahControlThread::notify_heap_changed() {
534 // This is called from allocation path, and thus should be fast.
535
536 // Update monitoring counters when we took a new region. This amortizes the
537 // update costs on slow path.
538 if (_do_counters_update.is_unset()) {
539 _do_counters_update.set();
540 }
541 // Notify that something had changed.
542 if (_heap_changed.is_unset()) {
543 _heap_changed.set();
544 }
545 }
546
547 void ShenandoahControlThread::pacing_notify_alloc(size_t words) {
548 assert(ShenandoahPacing, "should only call when pacing is enabled");
549 Atomic::add(words, &_allocs_seen);
550 }
551
552 void ShenandoahControlThread::set_forced_counters_update(bool value) {
553 _force_counters_update.set_cond(value);
554 }
555
556 void ShenandoahControlThread::print() const {
557 print_on(tty);
558 }
559
560 void ShenandoahControlThread::print_on(outputStream* st) const {
561 st->print("Shenandoah Concurrent Thread");
562 Thread::print_on(st);
563 st->cr();
564 }
565
566 void ShenandoahControlThread::start() {
567 create_and_start();
568 }
569
570 void ShenandoahControlThread::makeSurrogateLockerThread(TRAPS) {
571 assert(UseShenandoahGC, "SLT thread needed only for concurrent GC");
572 assert(THREAD->is_Java_thread(), "must be a Java thread");
573 assert(_slt == NULL, "SLT already created");
574 _slt = SurrogateLockerThread::make(THREAD);
575 }
576
577 void ShenandoahControlThread::prepare_for_graceful_shutdown() {
578 _graceful_shutdown.set();
579 }
580
581 bool ShenandoahControlThread::in_graceful_shutdown() {
582 return _graceful_shutdown.is_set();
583 }