1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 
  27 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  29 #include "gc/shenandoah/shenandoahControlThread.hpp"
  30 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  31 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  33 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  34 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  35 #include "gc/shenandoah/shenandoahUtils.hpp"
  36 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  37 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  38 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  39 #include "memory/iterator.hpp"
  40 #include "memory/universe.hpp"
  41 #include "runtime/atomic.hpp"
  42 
  43 ShenandoahControlThread::ShenandoahControlThread() :
  44   ConcurrentGCThread(),
  45   _alloc_failure_waiters_lock(Mutex::leaf, "ShenandoahAllocFailureGC_lock", true, Monitor::_safepoint_check_always),
  46   _gc_waiters_lock(Mutex::leaf, "ShenandoahRequestedGC_lock", true, Monitor::_safepoint_check_always),
  47   _periodic_task(this),
  48   _requested_gc_cause(GCCause::_no_cause_specified),
  49   _degen_point(ShenandoahHeap::_degenerated_outside_cycle),
  50   _allocs_seen(0) {
  51 
  52   reset_gc_id();
  53   create_and_start(ShenandoahCriticalControlThreadPriority ? CriticalPriority : NearMaxPriority);
  54   _periodic_task.enroll();
  55   _periodic_satb_flush_task.enroll();
  56   if (ShenandoahPacing) {
  57     _periodic_pacer_notify_task.enroll();
  58   }
  59 }
  60 
  61 ShenandoahControlThread::~ShenandoahControlThread() {
  62   // This is here so that super is called.
  63 }
  64 
  65 void ShenandoahPeriodicTask::task() {
  66   _thread->handle_force_counters_update();
  67   _thread->handle_counters_update();
  68 }
  69 
  70 void ShenandoahPeriodicSATBFlushTask::task() {
  71   ShenandoahHeap::heap()->force_satb_flush_all_threads();
  72 }
  73 
  74 void ShenandoahPeriodicPacerNotify::task() {
  75   assert(ShenandoahPacing, "Should not be here otherwise");
  76   ShenandoahHeap::heap()->pacer()->notify_waiters();
  77 }
  78 
  79 void ShenandoahControlThread::run_service() {
  80   ShenandoahHeap* heap = ShenandoahHeap::heap();
  81 
  82   GCMode default_mode = concurrent_normal;
  83   GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc;
  84   int sleep = ShenandoahControlIntervalMin;
  85 
  86   double last_shrink_time = os::elapsedTime();
  87   double last_sleep_adjust_time = os::elapsedTime();
  88 
  89   // Shrink period avoids constantly polling regions for shrinking.
  90   // Having a period 10x lower than the delay would mean we hit the
  91   // shrinking with lag of less than 1/10-th of true delay.
  92   // ShenandoahUncommitDelay is in msecs, but shrink_period is in seconds.
  93   double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10;
  94 
  95   ShenandoahCollectorPolicy* policy = heap->shenandoah_policy();
  96   ShenandoahHeuristics* heuristics = heap->heuristics();
  97   while (!in_graceful_shutdown() && !should_terminate()) {
  98     // Figure out if we have pending requests.
  99     bool alloc_failure_pending = _alloc_failure_gc.is_set();
 100     bool explicit_gc_requested = _gc_requested.is_set() &&  is_explicit_gc(_requested_gc_cause);
 101     bool implicit_gc_requested = _gc_requested.is_set() && !is_explicit_gc(_requested_gc_cause);
 102 
 103     // This control loop iteration have seen this much allocations.
 104     size_t allocs_seen = Atomic::xchg(&_allocs_seen, (size_t)0);
 105 
 106     // Choose which GC mode to run in. The block below should select a single mode.
 107     GCMode mode = none;
 108     GCCause::Cause cause = GCCause::_last_gc_cause;
 109     ShenandoahHeap::ShenandoahDegenPoint degen_point = ShenandoahHeap::_degenerated_unset;
 110 
 111     if (alloc_failure_pending) {
 112       // Allocation failure takes precedence: we have to deal with it first thing
 113       log_info(gc)("Trigger: Handle Allocation Failure");
 114 
 115       cause = GCCause::_allocation_failure;
 116 
 117       // Consume the degen point, and seed it with default value
 118       degen_point = _degen_point;
 119       _degen_point = ShenandoahHeap::_degenerated_outside_cycle;
 120 
 121       if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle()) {
 122         heuristics->record_allocation_failure_gc();
 123         policy->record_alloc_failure_to_degenerated(degen_point);
 124         mode = stw_degenerated;
 125       } else {
 126         heuristics->record_allocation_failure_gc();
 127         policy->record_alloc_failure_to_full();
 128         mode = stw_full;
 129       }
 130 
 131     } else if (explicit_gc_requested) {
 132       cause = _requested_gc_cause;
 133       log_info(gc)("Trigger: Explicit GC request (%s)", GCCause::to_string(cause));
 134 
 135       heuristics->record_requested_gc();
 136 
 137       if (ExplicitGCInvokesConcurrent) {
 138         policy->record_explicit_to_concurrent();
 139         mode = default_mode;
 140         // Unload and clean up everything
 141         heap->set_process_references(heuristics->can_process_references());
 142         heap->set_unload_classes(heuristics->can_unload_classes());
 143       } else {
 144         policy->record_explicit_to_full();
 145         mode = stw_full;
 146       }
 147     } else if (implicit_gc_requested) {
 148       cause = _requested_gc_cause;
 149       log_info(gc)("Trigger: Implicit GC request (%s)", GCCause::to_string(cause));
 150 
 151       heuristics->record_requested_gc();
 152 
 153       if (ShenandoahImplicitGCInvokesConcurrent) {
 154         policy->record_implicit_to_concurrent();
 155         mode = default_mode;
 156 
 157         // Unload and clean up everything
 158         heap->set_process_references(heuristics->can_process_references());
 159         heap->set_unload_classes(heuristics->can_unload_classes());
 160       } else {
 161         policy->record_implicit_to_full();
 162         mode = stw_full;
 163       }
 164     } else {
 165       // Potential normal cycle: ask heuristics if it wants to act
 166       if (heuristics->should_start_gc()) {
 167         mode = default_mode;
 168         cause = default_cause;
 169       }
 170 
 171       // Ask policy if this cycle wants to process references or unload classes
 172       heap->set_process_references(heuristics->should_process_references());
 173       heap->set_unload_classes(heuristics->should_unload_classes());
 174     }
 175 
 176     // Blow all soft references on this cycle, if handling allocation failure,
 177     // either implicit or explicit GC request,  or we are requested to do so unconditionally.
 178     if (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs) {
 179       heap->soft_ref_policy()->set_should_clear_all_soft_refs(true);
 180     }
 181 
 182     bool gc_requested = (mode != none);
 183     assert (!gc_requested || cause != GCCause::_last_gc_cause, "GC cause should be set");
 184 
 185     if (gc_requested) {
 186       // GC is starting, bump the internal ID
 187       update_gc_id();
 188 
 189       heap->reset_bytes_allocated_since_gc_start();
 190 
 191       // Use default constructor to snapshot the Metaspace state before GC.
 192       metaspace::MetaspaceSizesSnapshot meta_sizes;
 193 
 194       // If GC was requested, we are sampling the counters even without actual triggers
 195       // from allocation machinery. This captures GC phases more accurately.
 196       set_forced_counters_update(true);
 197 
 198       // If GC was requested, we better dump freeset data for performance debugging
 199       {
 200         ShenandoahHeapLocker locker(heap->lock());
 201         heap->free_set()->log_status();
 202       }
 203 
 204       switch (mode) {
 205         case concurrent_normal:
 206           service_concurrent_normal_cycle(cause);
 207           break;
 208         case stw_degenerated:
 209           service_stw_degenerated_cycle(cause, degen_point);
 210           break;
 211         case stw_full:
 212           service_stw_full_cycle(cause);
 213           break;
 214         default:
 215           ShouldNotReachHere();
 216       }
 217 
 218       // If this was the requested GC cycle, notify waiters about it
 219       if (explicit_gc_requested || implicit_gc_requested) {
 220         notify_gc_waiters();
 221       }
 222 
 223       // If this was the allocation failure GC cycle, notify waiters about it
 224       if (alloc_failure_pending) {
 225         notify_alloc_failure_waiters();
 226       }
 227 
 228       // Report current free set state at the end of cycle, whether
 229       // it is a normal completion, or the abort.
 230       {
 231         ShenandoahHeapLocker locker(heap->lock());
 232         heap->free_set()->log_status();
 233 
 234         // Notify Universe about new heap usage. This has implications for
 235         // global soft refs policy, and we better report it every time heap
 236         // usage goes down.
 237         Universe::update_heap_info_at_gc();
 238       }
 239 
 240       // Disable forced counters update, and update counters one more time
 241       // to capture the state at the end of GC session.
 242       handle_force_counters_update();
 243       set_forced_counters_update(false);
 244 
 245       // Retract forceful part of soft refs policy
 246       heap->soft_ref_policy()->set_should_clear_all_soft_refs(false);
 247 
 248       // Clear metaspace oom flag, if current cycle unloaded classes
 249       if (heap->unload_classes()) {
 250         heuristics->clear_metaspace_oom();
 251       }
 252 
 253       // Commit worker statistics to cycle data
 254       heap->phase_timings()->flush_par_workers_to_cycle();
 255       if (ShenandoahPacing) {
 256         heap->pacer()->flush_stats_to_cycle();
 257       }
 258 
 259       // Print GC stats for current cycle
 260       {
 261         LogTarget(Info, gc, stats) lt;
 262         if (lt.is_enabled()) {
 263           ResourceMark rm;
 264           LogStream ls(lt);
 265           heap->phase_timings()->print_cycle_on(&ls);
 266           if (ShenandoahPacing) {
 267             heap->pacer()->print_cycle_on(&ls);
 268           }
 269         }
 270       }
 271 
 272       // Commit statistics to globals
 273       heap->phase_timings()->flush_cycle_to_global();
 274 
 275       // Print Metaspace change following GC (if logging is enabled).
 276       MetaspaceUtils::print_metaspace_change(meta_sizes);
 277 
 278       // GC is over, we are at idle now
 279       if (ShenandoahPacing) {
 280         heap->pacer()->setup_for_idle();
 281       }
 282     } else {
 283       // Allow allocators to know we have seen this much regions
 284       if (ShenandoahPacing && (allocs_seen > 0)) {
 285         heap->pacer()->report_alloc(allocs_seen);
 286       }
 287     }
 288 
 289     double current = os::elapsedTime();
 290 
 291     if (ShenandoahUncommit && (explicit_gc_requested || (current - last_shrink_time > shrink_period))) {
 292       // Try to uncommit enough stale regions. Explicit GC tries to uncommit everything.
 293       // Regular paths uncommit only occasionally.
 294       double shrink_before = explicit_gc_requested ?
 295                              current :
 296                              current - (ShenandoahUncommitDelay / 1000.0);
 297       service_uncommit(shrink_before);
 298       heap->phase_timings()->flush_cycle_to_global();
 299       last_shrink_time = current;
 300     }
 301 
 302     // Wait before performing the next action. If allocation happened during this wait,
 303     // we exit sooner, to let heuristics re-evaluate new conditions. If we are at idle,
 304     // back off exponentially.
 305     if (_heap_changed.try_unset()) {
 306       sleep = ShenandoahControlIntervalMin;
 307     } else if ((current - last_sleep_adjust_time) * 1000 > ShenandoahControlIntervalAdjustPeriod){
 308       sleep = MIN2<int>(ShenandoahControlIntervalMax, MAX2(1, sleep * 2));
 309       last_sleep_adjust_time = current;
 310     }
 311     os::naked_short_sleep(sleep);
 312   }
 313 
 314   // Wait for the actual stop(), can't leave run_service() earlier.
 315   while (!should_terminate()) {
 316     os::naked_short_sleep(ShenandoahControlIntervalMin);
 317   }
 318 }
 319 
 320 void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cause) {
 321   // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during
 322   // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there.
 323   // If second allocation failure happens during Degenerated GC cycle (for example, when GC
 324   // tries to evac something and no memory is available), cycle degrades to Full GC.
 325   //
 326   // There are also a shortcut through the normal cycle: immediate garbage shortcut, when
 327   // heuristics says there are no regions to compact, and all the collection comes from immediately
 328   // reclaimable regions.
 329   //
 330   // ................................................................................................
 331   //
 332   //                                    (immediate garbage shortcut)                Concurrent GC
 333   //                             /-------------------------------------------\
 334   //                             |                                           |
 335   //                             |                                           |
 336   //                             |                                           |
 337   //                             |                                           v
 338   // [START] ----> Conc Mark ----o----> Conc Evac --o--> Conc Update-Refs ---o----> [END]
 339   //                   |                    |                 |              ^
 340   //                   | (af)               | (af)            | (af)         |
 341   // ..................|....................|.................|..............|.......................
 342   //                   |                    |                 |              |
 343   //                   |                    |                 |              |      Degenerated GC
 344   //                   v                    v                 v              |
 345   //               STW Mark ----------> STW Evac ----> STW Update-Refs ----->o
 346   //                   |                    |                 |              ^
 347   //                   | (af)               | (af)            | (af)         |
 348   // ..................|....................|.................|..............|.......................
 349   //                   |                    |                 |              |
 350   //                   |                    v                 |              |      Full GC
 351   //                   \------------------->o<----------------/              |
 352   //                                        |                                |
 353   //                                        v                                |
 354   //                                      Full GC  --------------------------/
 355   //
 356   ShenandoahHeap* heap = ShenandoahHeap::heap();
 357 
 358   if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_outside_cycle)) return;
 359 
 360   GCIdMark gc_id_mark;
 361   ShenandoahGCSession session(cause);
 362 
 363   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 364 
 365   // Reset for upcoming marking
 366   heap->entry_reset();
 367 
 368   // Start initial mark under STW
 369   heap->vmop_entry_init_mark();
 370 
 371   // Continue concurrent mark
 372   heap->entry_mark();
 373   if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_mark)) return;
 374 
 375   // If not cancelled, can try to concurrently pre-clean
 376   heap->entry_preclean();
 377 
 378   // Complete marking under STW, and start evacuation
 379   heap->vmop_entry_final_mark();
 380 
 381   // Process weak roots that might still point to regions that would be broken by cleanup
 382   if (heap->is_concurrent_weak_root_in_progress()) {
 383     heap->entry_weak_roots();
 384   }
 385 
 386   // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
 387   // the space. This would be the last action if there is nothing to evacuate.
 388   heap->entry_cleanup_early();
 389 
 390   {
 391     ShenandoahHeapLocker locker(heap->lock());
 392     heap->free_set()->log_status();
 393   }
 394 
 395   // Perform concurrent class unloading
 396   if (heap->is_concurrent_weak_root_in_progress()) {
 397     heap->entry_class_unloading();
 398   }
 399 
 400   // Processing strong roots
 401   // This may be skipped if there is nothing to update/evacuate.
 402   // If so, strong_root_in_progress would be unset.
 403   if (heap->is_concurrent_strong_root_in_progress()) {
 404     heap->entry_strong_roots();
 405   }
 406 
 407   // Continue the cycle with evacuation and optional update-refs.
 408   // This may be skipped if there is nothing to evacuate.
 409   // If so, evac_in_progress would be unset by collection set preparation code.
 410   if (heap->is_evacuation_in_progress()) {
 411     // Concurrently evacuate
 412     heap->entry_evac();
 413     if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_evac)) return;
 414 
 415     // Perform update-refs phase.
 416     heap->vmop_entry_init_updaterefs();
 417     heap->entry_updaterefs();
 418     if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_updaterefs)) return;
 419 
 420     heap->vmop_entry_final_updaterefs();
 421 
 422     // Update references freed up collection set, kick the cleanup to reclaim the space.
 423     heap->entry_cleanup_complete();
 424   }
 425 
 426   // Cycle is complete
 427   heap->heuristics()->record_success_concurrent();
 428   heap->shenandoah_policy()->record_success_concurrent();
 429 }
 430 
 431 bool ShenandoahControlThread::check_cancellation_or_degen(ShenandoahHeap::ShenandoahDegenPoint point) {
 432   ShenandoahHeap* heap = ShenandoahHeap::heap();
 433   if (heap->cancelled_gc()) {
 434     assert (is_alloc_failure_gc() || in_graceful_shutdown(), "Cancel GC either for alloc failure GC, or gracefully exiting");
 435     if (!in_graceful_shutdown()) {
 436       assert (_degen_point == ShenandoahHeap::_degenerated_outside_cycle,
 437               "Should not be set yet: %s", ShenandoahHeap::degen_point_to_string(_degen_point));
 438       _degen_point = point;
 439     }
 440     return true;
 441   }
 442   return false;
 443 }
 444 
 445 void ShenandoahControlThread::stop_service() {
 446   // Nothing to do here.
 447 }
 448 
 449 void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) {
 450   GCIdMark gc_id_mark;
 451   ShenandoahGCSession session(cause);
 452 
 453   ShenandoahHeap* heap = ShenandoahHeap::heap();
 454   heap->vmop_entry_full(cause);
 455 
 456   heap->heuristics()->record_success_full();
 457   heap->shenandoah_policy()->record_success_full();
 458 }
 459 
 460 void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahHeap::ShenandoahDegenPoint point) {
 461   assert (point != ShenandoahHeap::_degenerated_unset, "Degenerated point should be set");
 462 
 463   GCIdMark gc_id_mark;
 464   ShenandoahGCSession session(cause);
 465 
 466   ShenandoahHeap* heap = ShenandoahHeap::heap();
 467   heap->vmop_degenerated(point);
 468 
 469   heap->heuristics()->record_success_degenerated();
 470   heap->shenandoah_policy()->record_success_degenerated();
 471 }
 472 
 473 void ShenandoahControlThread::service_uncommit(double shrink_before) {
 474   ShenandoahHeap* heap = ShenandoahHeap::heap();
 475 
 476   // Determine if there is work to do. This avoids taking heap lock if there is
 477   // no work available, avoids spamming logs with superfluous logging messages,
 478   // and minimises the amount of work while locks are taken.
 479 
 480   if (heap->committed() <= heap->min_capacity()) return;
 481 
 482   bool has_work = false;
 483   for (size_t i = 0; i < heap->num_regions(); i++) {
 484     ShenandoahHeapRegion *r = heap->get_region(i);
 485     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 486       has_work = true;
 487       break;
 488     }
 489   }
 490 
 491   if (has_work) {
 492     heap->entry_uncommit(shrink_before);
 493   }
 494 }
 495 
 496 bool ShenandoahControlThread::is_explicit_gc(GCCause::Cause cause) const {
 497   return GCCause::is_user_requested_gc(cause) ||
 498          GCCause::is_serviceability_requested_gc(cause);
 499 }
 500 
 501 void ShenandoahControlThread::request_gc(GCCause::Cause cause) {
 502   assert(GCCause::is_user_requested_gc(cause) ||
 503          GCCause::is_serviceability_requested_gc(cause) ||
 504          cause == GCCause::_metadata_GC_clear_soft_refs ||
 505          cause == GCCause::_full_gc_alot ||
 506          cause == GCCause::_wb_full_gc ||
 507          cause == GCCause::_scavenge_alot,
 508          "only requested GCs here");
 509 
 510   if (is_explicit_gc(cause)) {
 511     if (!DisableExplicitGC) {
 512       handle_requested_gc(cause);
 513     }
 514   } else {
 515     handle_requested_gc(cause);
 516   }
 517 }
 518 
 519 void ShenandoahControlThread::handle_requested_gc(GCCause::Cause cause) {
 520   // Make sure we have at least one complete GC cycle before unblocking
 521   // from the explicit GC request.
 522   //
 523   // This is especially important for weak references cleanup and/or native
 524   // resources (e.g. DirectByteBuffers) machinery: when explicit GC request
 525   // comes very late in the already running cycle, it would miss lots of new
 526   // opportunities for cleanup that were made available before the caller
 527   // requested the GC.
 528 
 529   MonitorLocker ml(&_gc_waiters_lock);
 530   size_t current_gc_id = get_gc_id();
 531   size_t required_gc_id = current_gc_id + 1;
 532   while (current_gc_id < required_gc_id) {
 533     _gc_requested.set();
 534     _requested_gc_cause = cause;
 535     ml.wait();
 536     current_gc_id = get_gc_id();
 537   }
 538 }
 539 
 540 void ShenandoahControlThread::handle_alloc_failure(ShenandoahAllocRequest& req) {
 541   ShenandoahHeap* heap = ShenandoahHeap::heap();
 542 
 543   assert(current()->is_Java_thread(), "expect Java thread here");
 544 
 545   if (try_set_alloc_failure_gc()) {
 546     // Only report the first allocation failure
 547     log_info(gc)("Failed to allocate %s, " SIZE_FORMAT "%s",
 548                  req.type_string(),
 549                  byte_size_in_proper_unit(req.size() * HeapWordSize), proper_unit_for_byte_size(req.size() * HeapWordSize));
 550 
 551     // Now that alloc failure GC is scheduled, we can abort everything else
 552     heap->cancel_gc(GCCause::_allocation_failure);
 553   }
 554 
 555   MonitorLocker ml(&_alloc_failure_waiters_lock);
 556   while (is_alloc_failure_gc()) {
 557     ml.wait();
 558   }
 559 }
 560 
 561 void ShenandoahControlThread::handle_alloc_failure_evac(size_t words) {
 562   ShenandoahHeap* heap = ShenandoahHeap::heap();
 563 
 564   if (try_set_alloc_failure_gc()) {
 565     // Only report the first allocation failure
 566     log_info(gc)("Failed to allocate " SIZE_FORMAT "%s for evacuation",
 567                  byte_size_in_proper_unit(words * HeapWordSize), proper_unit_for_byte_size(words * HeapWordSize));
 568   }
 569 
 570   // Forcefully report allocation failure
 571   heap->cancel_gc(GCCause::_shenandoah_allocation_failure_evac);
 572 }
 573 
 574 void ShenandoahControlThread::notify_alloc_failure_waiters() {
 575   _alloc_failure_gc.unset();
 576   MonitorLocker ml(&_alloc_failure_waiters_lock);
 577   ml.notify_all();
 578 }
 579 
 580 bool ShenandoahControlThread::try_set_alloc_failure_gc() {
 581   return _alloc_failure_gc.try_set();
 582 }
 583 
 584 bool ShenandoahControlThread::is_alloc_failure_gc() {
 585   return _alloc_failure_gc.is_set();
 586 }
 587 
 588 void ShenandoahControlThread::notify_gc_waiters() {
 589   _gc_requested.unset();
 590   MonitorLocker ml(&_gc_waiters_lock);
 591   ml.notify_all();
 592 }
 593 
 594 void ShenandoahControlThread::handle_counters_update() {
 595   if (_do_counters_update.is_set()) {
 596     _do_counters_update.unset();
 597     ShenandoahHeap::heap()->monitoring_support()->update_counters();
 598   }
 599 }
 600 
 601 void ShenandoahControlThread::handle_force_counters_update() {
 602   if (_force_counters_update.is_set()) {
 603     _do_counters_update.unset(); // reset these too, we do update now!
 604     ShenandoahHeap::heap()->monitoring_support()->update_counters();
 605   }
 606 }
 607 
 608 void ShenandoahControlThread::notify_heap_changed() {
 609   // This is called from allocation path, and thus should be fast.
 610 
 611   // Update monitoring counters when we took a new region. This amortizes the
 612   // update costs on slow path.
 613   if (_do_counters_update.is_unset()) {
 614     _do_counters_update.set();
 615   }
 616   // Notify that something had changed.
 617   if (_heap_changed.is_unset()) {
 618     _heap_changed.set();
 619   }
 620 }
 621 
 622 void ShenandoahControlThread::pacing_notify_alloc(size_t words) {
 623   assert(ShenandoahPacing, "should only call when pacing is enabled");
 624   Atomic::add(&_allocs_seen, words);
 625 }
 626 
 627 void ShenandoahControlThread::set_forced_counters_update(bool value) {
 628   _force_counters_update.set_cond(value);
 629 }
 630 
 631 void ShenandoahControlThread::reset_gc_id() {
 632   Atomic::store(&_gc_id, (size_t)0);
 633 }
 634 
 635 void ShenandoahControlThread::update_gc_id() {
 636   Atomic::inc(&_gc_id);
 637 }
 638 
 639 size_t ShenandoahControlThread::get_gc_id() {
 640   return Atomic::load(&_gc_id);
 641 }
 642 
 643 void ShenandoahControlThread::print() const {
 644   print_on(tty);
 645 }
 646 
 647 void ShenandoahControlThread::print_on(outputStream* st) const {
 648   st->print("Shenandoah Concurrent Thread");
 649   Thread::print_on(st);
 650   st->cr();
 651 }
 652 
 653 void ShenandoahControlThread::start() {
 654   create_and_start();
 655 }
 656 
 657 void ShenandoahControlThread::prepare_for_graceful_shutdown() {
 658   _graceful_shutdown.set();
 659 }
 660 
 661 bool ShenandoahControlThread::in_graceful_shutdown() {
 662   return _graceful_shutdown.is_set();
 663 }