--- old/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp 2018-05-08 09:19:20.052061994 +0200 +++ new/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp 2018-05-08 09:19:19.762053048 +0200 @@ -1706,6 +1706,44 @@ } } +class G1PrecleanYieldClosure : public YieldClosure { + G1ConcurrentMark* _cm; + +public: + G1PrecleanYieldClosure(G1ConcurrentMark* cm) : _cm(cm) { } + + virtual bool should_return() { + return _cm->has_aborted(); + } + + virtual bool should_return_fine_grain() { + _cm->do_yield_check(); + return _cm->has_aborted(); + } +}; + +void G1ConcurrentMark::preclean() { + assert(G1UseReferencePrecleaning, "Precleaning must be enabled."); + + SuspendibleThreadSetJoiner joiner; + + G1CMKeepAliveAndDrainClosure keep_alive(this, task(0), true /* is_serial */); + G1CMDrainMarkingStackClosure drain_mark_stack(this, task(0), true /* is_serial */); + + set_concurrency_and_phase(1, true); + + G1PrecleanYieldClosure yield_cl(this); + + ReferenceProcessor* rp = _g1h->ref_processor_cm(); + // Precleaning is single threaded. Temporarily disable MT discovery. + ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(rp, false); + rp->preclean_discovered_references(rp->is_alive_non_header(), + &keep_alive, + &drain_mark_stack, + &yield_cl, + _gc_timer_cm); +} + // When sampling object counts, we already swapped the mark bitmaps, so we need to use // the prev bitmap determining liveness. class G1ObjectCountIsAliveClosure: public BoolObjectClosure { --- old/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp 2018-05-08 09:19:21.266099440 +0200 +++ new/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp 2018-05-08 09:19:20.979090588 +0200 @@ -562,6 +562,9 @@ // Do concurrent phase of marking, to a tentative transitive closure. void mark_from_roots(); + // Do concurrent preclean work. + void preclean(); + void remark(); void cleanup(); --- old/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp 2018-05-08 09:19:22.449135931 +0200 +++ new/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp 2018-05-08 09:19:22.160127016 +0200 @@ -57,6 +57,7 @@ expander(SCAN_ROOT_REGIONS,, "Concurrent Scan Root Regions") \ expander(CONCURRENT_MARK,, "Concurrent Mark") \ expander(MARK_FROM_ROOTS,, "Concurrent Mark From Roots") \ + expander(PRECLEAN,, "Concurrent Preclean") \ expander(BEFORE_REMARK,, NULL) \ expander(REMARK,, NULL) \ expander(REBUILD_REMEMBERED_SETS,, "Concurrent Rebuild Remembered Sets") \ @@ -309,7 +310,12 @@ break; } - // Provide a control point after mark_from_roots. + if (G1UseReferencePrecleaning) { + G1ConcPhase p(G1ConcurrentPhase::PRECLEAN, this); + _cm->preclean(); + } + + // Provide a control point before remark. { G1ConcPhaseManager p(G1ConcurrentPhase::BEFORE_REMARK, this); } --- old/src/hotspot/share/gc/g1/g1_globals.hpp 2018-05-08 09:19:23.620172051 +0200 +++ new/src/hotspot/share/gc/g1/g1_globals.hpp 2018-05-08 09:19:23.334163229 +0200 @@ -79,6 +79,10 @@ "draining concurrent marking work queues.") \ range(1, INT_MAX) \ \ + experimental(bool, G1UseReferencePrecleaning, true, \ + "Concurrently preclean java.lang.ref.references instances " \ + "before the Remark pause.") \ + \ experimental(double, G1LastPLABAverageOccupancy, 50.0, \ "The expected average occupancy of the last PLAB in " \ "percent.") \ --- old/src/hotspot/share/gc/shared/referenceProcessor.cpp 2018-05-08 09:19:24.794208264 +0200 +++ new/src/hotspot/share/gc/shared/referenceProcessor.cpp 2018-05-08 09:19:24.505199350 +0200 @@ -594,19 +594,33 @@ bool _clear_referent; }; +void ReferenceProcessor::log_reflist(const char* prefix, DiscoveredList list[], uint num_active_queues) { + LogTarget(Trace, gc, ref) lt; + + if (!lt.is_enabled()) { + return; + } + + size_t total = 0; + + LogStream ls(lt); + ls.print("%s", prefix); + for (uint i = 0; i < num_active_queues; i++) { + ls.print(SIZE_FORMAT " ", list[i].length()); + total += list[i].length(); + } + ls.print_cr("(" SIZE_FORMAT ")", total); +} + #ifndef PRODUCT -void ReferenceProcessor::log_reflist_counts(DiscoveredList ref_lists[], uint active_length, size_t total_refs) { +void ReferenceProcessor::log_reflist_counts(DiscoveredList ref_lists[], uint num_active_queues) { if (!log_is_enabled(Trace, gc, ref)) { return; } - stringStream st; - for (uint i = 0; i < active_length; ++i) { - st.print(SIZE_FORMAT " ", ref_lists[i].length()); - } - log_develop_trace(gc, ref)("%s= " SIZE_FORMAT, st.as_string(), total_refs); + log_reflist("", ref_lists, num_active_queues); #ifdef ASSERT - for (uint i = active_length; i < _max_num_queues; i++) { + for (uint i = num_active_queues; i < _max_num_queues; i++) { assert(ref_lists[i].length() == 0, SIZE_FORMAT " unexpected References in %u", ref_lists[i].length(), i); } @@ -629,10 +643,11 @@ size_t total_refs = 0; log_develop_trace(gc, ref)("Balance ref_lists "); + log_reflist_counts(ref_lists, _max_num_queues); + for (uint i = 0; i < _max_num_queues; ++i) { total_refs += ref_lists[i].length(); } - log_reflist_counts(ref_lists, _max_num_queues, total_refs); size_t avg_refs = total_refs / _num_queues + 1; uint to_idx = 0; for (uint from_idx = 0; from_idx < _max_num_queues; from_idx++) { @@ -693,11 +708,11 @@ } } #ifdef ASSERT + log_reflist_counts(ref_lists, _num_queues); size_t balanced_total_refs = 0; for (uint i = 0; i < _num_queues; ++i) { balanced_total_refs += ref_lists[i].length(); } - log_reflist_counts(ref_lists, _num_queues, balanced_total_refs); assert(total_refs == balanced_total_refs, "Balancing was incomplete"); #endif } @@ -1011,63 +1026,79 @@ return false; } -// Preclean the discovered references by removing those -// whose referents are alive, and by marking from those that -// are not active. These lists can be handled here -// in any order and, indeed, concurrently. -void ReferenceProcessor::preclean_discovered_references( - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - YieldClosure* yield, - GCTimer* gc_timer) { - +void ReferenceProcessor::preclean_discovered_references(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + YieldClosure* yield, + GCTimer* gc_timer) { + // These lists can be handled here in any order and, indeed, concurrently. + // Soft references { GCTraceTime(Debug, gc, ref) tm("Preclean SoftReferences", gc_timer); + log_reflist("SoftRef before: ", _discoveredSoftRefs, _max_num_queues); for (uint i = 0; i < _max_num_queues; i++) { if (yield->should_return()) { return; + } + if (preclean_discovered_reflist(_discoveredSoftRefs[i], is_alive, + keep_alive, complete_gc, yield)) { + log_reflist("SoftRef abort: ", _discoveredSoftRefs, _max_num_queues); + return; } - preclean_discovered_reflist(_discoveredSoftRefs[i], is_alive, - keep_alive, complete_gc, yield); } + log_reflist("SoftRef after: ", _discoveredSoftRefs, _max_num_queues); } // Weak references { GCTraceTime(Debug, gc, ref) tm("Preclean WeakReferences", gc_timer); + log_reflist("WeakRef before: ", _discoveredWeakRefs, _max_num_queues); for (uint i = 0; i < _max_num_queues; i++) { if (yield->should_return()) { return; } - preclean_discovered_reflist(_discoveredWeakRefs[i], is_alive, - keep_alive, complete_gc, yield); + if (preclean_discovered_reflist(_discoveredWeakRefs[i], is_alive, + keep_alive, complete_gc, yield)) { + log_reflist("WeakRef abort: ", _discoveredWeakRefs, _max_num_queues); + return; + } } + log_reflist("WeakRef after: ", _discoveredWeakRefs, _max_num_queues); } // Final references { GCTraceTime(Debug, gc, ref) tm("Preclean FinalReferences", gc_timer); + log_reflist("FinalRef before: ", _discoveredFinalRefs, _max_num_queues); for (uint i = 0; i < _max_num_queues; i++) { if (yield->should_return()) { return; } - preclean_discovered_reflist(_discoveredFinalRefs[i], is_alive, - keep_alive, complete_gc, yield); + if (preclean_discovered_reflist(_discoveredFinalRefs[i], is_alive, + keep_alive, complete_gc, yield)) { + log_reflist("FinalRef abort: ", _discoveredFinalRefs, _max_num_queues); + return; + } } + log_reflist("FinalRef after: ", _discoveredFinalRefs, _max_num_queues); } // Phantom references { GCTraceTime(Debug, gc, ref) tm("Preclean PhantomReferences", gc_timer); + log_reflist("PhantomRef before: ", _discoveredPhantomRefs, _max_num_queues); for (uint i = 0; i < _max_num_queues; i++) { if (yield->should_return()) { return; } - preclean_discovered_reflist(_discoveredPhantomRefs[i], is_alive, - keep_alive, complete_gc, yield); + if (preclean_discovered_reflist(_discoveredPhantomRefs[i], is_alive, + keep_alive, complete_gc, yield)) { + log_reflist("PhantomRef abort: ", _discoveredPhantomRefs, _max_num_queues); + return; + } } + log_reflist("PhantomRef after: ", _discoveredPhantomRefs, _max_num_queues); } } @@ -1079,19 +1110,20 @@ // java.lang.Reference. As a result, we need to be careful below // that ref removal steps interleave safely with ref discovery steps // (in this thread). -void -ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_list, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - YieldClosure* yield) { +bool ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + YieldClosure* yield) { DiscoveredListIterator iter(refs_list, keep_alive, is_alive); while (iter.has_next()) { + if (yield->should_return_fine_grain()) { + return true; + } iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); oop obj = iter.obj(); oop next = java_lang_ref_Reference::next(obj); - if (iter.referent() == NULL || iter.is_referent_alive() || - next != NULL) { + if (iter.referent() == NULL || iter.is_referent_alive() || next != NULL) { // The referent has been cleared, or is alive, or the Reference is not // active; we need to trace and mark its cohort. log_develop_trace(gc, ref)("Precleaning Reference (" INTPTR_FORMAT ": %s)", @@ -1121,6 +1153,7 @@ iter.removed(), iter.processed(), p2i(&refs_list)); } ) + return false; } const char* ReferenceProcessor::list_name(uint i) { --- old/src/hotspot/share/gc/shared/referenceProcessor.hpp 2018-05-08 09:19:25.977244755 +0200 +++ new/src/hotspot/share/gc/shared/referenceProcessor.hpp 2018-05-08 09:19:25.692235964 +0200 @@ -279,15 +279,15 @@ OopClosure* keep_alive, VoidClosure* complete_gc); - // "Preclean" all the discovered reference lists - // by removing references with strongly reachable referents. + // "Preclean" all the discovered reference lists by removing references that + // are active (e.g. due to the mutator calling enqueue()) or with NULL or + // strongly reachable referents. // The first argument is a predicate on an oop that indicates - // its (strong) reachability and the second is a closure that + // its (strong) reachability and the fourth is a closure that // may be used to incrementalize or abort the precleaning process. // The caller is responsible for taking care of potential // interference with concurrent operations on these lists - // (or predicates involved) by other threads. Currently - // only used by the CMS collector. + // (or predicates involved) by other threads. void preclean_discovered_references(BoolObjectClosure* is_alive, OopClosure* keep_alive, VoidClosure* complete_gc, @@ -298,15 +298,17 @@ // occupying the i / _num_queues slot. const char* list_name(uint i); - // "Preclean" the given discovered reference list - // by removing references with strongly reachable referents. - // Currently used in support of CMS only. - void preclean_discovered_reflist(DiscoveredList& refs_list, +private: + // "Preclean" the given discovered reference list by removing references with + // the attributes mentioned in preclean_discovered_references(). + // Supports both normal and fine grain yielding. + // Returns whether the operation should be aborted. + bool preclean_discovered_reflist(DiscoveredList& refs_list, BoolObjectClosure* is_alive, OopClosure* keep_alive, VoidClosure* complete_gc, YieldClosure* yield); -private: + // round-robin mod _num_queues (not: _not_ mod _max_num_queues) uint next_id() { uint id = _next_id; @@ -323,7 +325,8 @@ void clear_discovered_references(DiscoveredList& refs_list); - void log_reflist_counts(DiscoveredList ref_lists[], uint active_length, size_t total_count) PRODUCT_RETURN; + void log_reflist(const char* prefix, DiscoveredList list[], uint num_active_queues); + void log_reflist_counts(DiscoveredList ref_lists[], uint num_active_queues) PRODUCT_RETURN; // Balances reference queues. void balance_queues(DiscoveredList ref_lists[]); --- old/src/hotspot/share/memory/iterator.hpp 2018-05-08 09:19:27.144280752 +0200 +++ new/src/hotspot/share/memory/iterator.hpp 2018-05-08 09:19:26.858271930 +0200 @@ -318,8 +318,10 @@ // by means of checking the return value from the polling // call. class YieldClosure : public StackObj { - public: - virtual bool should_return() = 0; +public: + virtual bool should_return() = 0; + // Yield on a fine-grain level. The check in case of not yielding should be very fast. + virtual bool should_return_fine_grain() { return false; } }; // Abstract closure for serializing data (read or write). --- old/test/hotspot/jtreg/gc/concurrent_phase_control/TestConcurrentPhaseControlG1.java 2018-05-08 09:19:28.353318044 +0200 +++ new/test/hotspot/jtreg/gc/concurrent_phase_control/TestConcurrentPhaseControlG1.java 2018-05-08 09:19:28.046308574 +0200 @@ -51,6 +51,7 @@ {"CONCURRENT_MARK", "Concurrent Mark [^FR]"}, {"IDLE", null}, // Resume IDLE before testing subphases {"MARK_FROM_ROOTS", "Concurrent Mark From Roots"}, + {"PRECLEAN", "Concurrent Preclean"}, {"BEFORE_REMARK", null}, {"REMARK", "Pause Remark"}, {"REBUILD_REMEMBERED_SETS", "Concurrent Rebuild Remembered Sets"}, --- old/test/hotspot/jtreg/gc/concurrent_phase_control/TestConcurrentPhaseControlG1Basics.java 2018-05-08 09:19:29.520354041 +0200 +++ new/test/hotspot/jtreg/gc/concurrent_phase_control/TestConcurrentPhaseControlG1Basics.java 2018-05-08 09:19:29.213344571 +0200 @@ -51,6 +51,7 @@ "SCAN_ROOT_REGIONS", "CONCURRENT_MARK", "MARK_FROM_ROOTS", + "PRECLEAN", "BEFORE_REMARK", "REMARK", "REBUILD_REMEMBERED_SETS",