--- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp 2019-11-11 16:09:55.319342976 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp 2019-11-11 16:09:55.233342981 +0100 @@ -51,6 +51,7 @@ #include "gc_implementation/shenandoah/shenandoahPassiveMode.hpp" #include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp" #include "gc_implementation/shenandoah/shenandoahTaskqueue.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalMode.hpp" #include "gc_implementation/shenandoah/shenandoahUtils.hpp" #include "gc_implementation/shenandoah/shenandoahVerifier.hpp" #include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp" @@ -357,6 +358,10 @@ _pacer = NULL; } + _traversal_gc = strcmp(ShenandoahGCMode, "traversal") == 0 ? + new ShenandoahTraversalGC(this, _num_regions) : + NULL; + _control_thread = new ShenandoahControlThread(); log_info(gc, init)("Initialize Shenandoah heap: " SIZE_FORMAT "%s initial, " SIZE_FORMAT "%s min, " SIZE_FORMAT "%s max", @@ -375,7 +380,9 @@ void ShenandoahHeap::initialize_heuristics() { if (ShenandoahGCMode != NULL) { - if (strcmp(ShenandoahGCMode, "normal") == 0) { + if (strcmp(ShenandoahGCMode, "traversal") == 0) { + _gc_mode = new ShenandoahTraversalMode(); + } else if (strcmp(ShenandoahGCMode, "normal") == 0) { _gc_mode = new ShenandoahNormalMode(); } else if (strcmp(ShenandoahGCMode, "passive") == 0) { _gc_mode = new ShenandoahPassiveMode(); @@ -409,6 +416,7 @@ _regions(NULL), _free_set(NULL), _collection_set(NULL), + _traversal_gc(NULL), _update_refs_iterator(this), _bytes_allocated_since_gc_start(0), _max_workers((uint)MAX2(ConcGCThreads, ParallelGCThreads)), @@ -431,6 +439,7 @@ log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial"); _scm = new ShenandoahConcurrentMark(); + _full_gc = new ShenandoahMarkCompact(); _used = 0; @@ -498,6 +507,7 @@ if (is_concurrent_mark_in_progress()) st->print("marking, "); if (is_evacuation_in_progress()) st->print("evacuating, "); if (is_update_refs_in_progress()) st->print("updating refs, "); + if (is_concurrent_traversal_in_progress()) st->print("traversal, "); if (is_degenerated_gc_in_progress()) st->print("degenerated gc, "); if (is_full_gc_in_progress()) st->print("full gc, "); if (is_full_gc_move_in_progress()) st->print("full gc move, "); @@ -1580,6 +1590,18 @@ concurrent_mark()->preclean_weak_refs(); } +void ShenandoahHeap::op_init_traversal() { + traversal_gc()->init_traversal_collection(); +} + +void ShenandoahHeap::op_traversal() { + traversal_gc()->concurrent_traversal_collection(); +} + +void ShenandoahHeap::op_final_traversal() { + traversal_gc()->final_traversal_collection(); +} + void ShenandoahHeap::op_full(GCCause::Cause cause) { ShenandoahMetricsSnapshot metrics; metrics.snap_before(); @@ -1608,6 +1630,24 @@ metrics.snap_before(); switch (point) { + case _degenerated_traversal: + { + // Drop the collection set. Note: this leaves some already forwarded objects + // behind, which may be problematic, see comments for ShenandoahEvacAssist + // workarounds in ShenandoahTraversalHeuristics. + + ShenandoahHeapLocker locker(lock()); + collection_set()->clear_current_index(); + for (size_t i = 0; i < collection_set()->count(); i++) { + ShenandoahHeapRegion* r = collection_set()->next(); + r->make_regular_bypass(); + } + collection_set()->clear(); + } + op_final_traversal(); + op_cleanup(); + return; + // The cases below form the Duff's-like device: it describes the actual GC cycle, // but enters it at different points, depending on which concurrent phase had // degenerated. @@ -1624,6 +1664,13 @@ set_process_references(heuristics()->can_process_references()); set_unload_classes(heuristics()->can_unload_classes()); + if (is_traversal_mode()) { + // Not possible to degenerate from here, upgrade to Full GC right away. + cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc); + op_degenerated_fail(); + return; + } + op_reset(); op_init_mark(); @@ -1753,7 +1800,7 @@ } void ShenandoahHeap::force_satb_flush_all_threads() { - if (!is_concurrent_mark_in_progress()) { + if (!is_concurrent_mark_in_progress() && !is_concurrent_traversal_in_progress()) { // No need to flush SATBs return; } @@ -1787,6 +1834,11 @@ JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress); } +void ShenandoahHeap::set_concurrent_traversal_in_progress(bool in_progress) { + set_gc_state_mask(TRAVERSAL | HAS_FORWARDED | UPDATEREFS, in_progress); + JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress); +} + void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint"); set_gc_state_mask(EVACUATION, in_progress); @@ -2310,6 +2362,26 @@ VMThread::execute(&op); } +void ShenandoahHeap::vmop_entry_init_traversal() { + TraceCollectorStats tcs(monitoring_support()->stw_collection_counters()); + ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc_gross); + + try_inject_alloc_failure(); + VM_ShenandoahInitTraversalGC op; + VMThread::execute(&op); +} + +void ShenandoahHeap::vmop_entry_final_traversal() { + TraceCollectorStats tcs(monitoring_support()->stw_collection_counters()); + ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc_gross); + + try_inject_alloc_failure(); + VM_ShenandoahFinalTraversalGC op; + VMThread::execute(&op); +} + void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) { TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters()); ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross); @@ -2398,6 +2470,36 @@ op_final_updaterefs(); } +void ShenandoahHeap::entry_init_traversal() { + ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc); + + static const char* msg = "Pause Init Traversal"; + GCTraceTime time(msg, PrintGC, _gc_timer, tracer()->gc_id()); + EventMark em("%s", msg); + + ShenandoahWorkerScope scope(workers(), + ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(), + "init traversal"); + + op_init_traversal(); +} + +void ShenandoahHeap::entry_final_traversal() { + ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc); + + static const char* msg = "Pause Final Traversal"; + GCTraceTime time(msg, PrintGC, _gc_timer, tracer()->gc_id()); + EventMark em("%s", msg); + + ShenandoahWorkerScope scope(workers(), + ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(), + "final traversal"); + + op_final_traversal(); +} + void ShenandoahHeap::entry_full(GCCause::Cause cause) { ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause); ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc); @@ -2523,6 +2625,21 @@ } } +void ShenandoahHeap::entry_traversal() { + static const char* msg = "Concurrent traversal"; + GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true); + EventMark em("%s", msg); + + TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters()); + + ShenandoahWorkerScope scope(workers(), + ShenandoahWorkerPolicy::calc_workers_for_conc_traversal(), + "concurrent traversal"); + + try_inject_alloc_failure(); + op_traversal(); +} + void ShenandoahHeap::entry_uncommit(double shrink_before) { static const char *msg = "Concurrent uncommit"; GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true); @@ -2651,6 +2768,8 @@ switch (point) { case _degenerated_unset: return "Pause Degenerated GC ()"; + case _degenerated_traversal: + return "Pause Degenerated GC (Traversal)"; case _degenerated_outside_cycle: return "Pause Degenerated GC (Outside of Cycle)"; case _degenerated_mark: