--- old/src/share/vm/gc/shared/referenceProcessor.cpp 2016-01-24 11:21:54.316758445 +0100 +++ new/src/share/vm/gc/shared/referenceProcessor.cpp 2016-01-24 11:21:54.251759564 +0100 @@ -117,9 +117,9 @@ } _discoveredSoftRefs = &_discovered_refs[0]; _discoveredWeakRefs = &_discoveredSoftRefs[_max_num_q]; - _discoveredFinalRefs = &_discoveredWeakRefs[_max_num_q]; + _discoveredEphemerons = &_discoveredWeakRefs[_max_num_q]; + _discoveredFinalRefs = &_discoveredEphemerons[_max_num_q]; _discoveredPhantomRefs = &_discoveredFinalRefs[_max_num_q]; - _discoveredCleanerRefs = &_discoveredPhantomRefs[_max_num_q]; // Initialize all entries to NULL for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { @@ -208,14 +208,22 @@ _soft_ref_timestamp_clock = java_lang_ref_SoftReference::clock(); - // Include cleaners in phantom statistics. We expect Cleaner - // references to be temporary, and don't want to deal with - // possible incompatibilities arising from making it more visible. ReferenceProcessorStats stats( total_count(_discoveredSoftRefs), total_count(_discoveredWeakRefs), + total_count(_discoveredEphemerons), total_count(_discoveredFinalRefs), - total_count(_discoveredPhantomRefs) + total_count(_discoveredCleanerRefs)); + total_count(_discoveredPhantomRefs)); + + // Ephemerons (phase2) before Soft references. This closes the hard-reachable + // set that arrises when the value of an ephemeron with a hard-reachable key + // refers to an otherwise weaker-than-hard-reachable key of some other ephemeron. + { + GCTraceTime(Debug, gc, ref) tt("Ephemeron-HardClosure", gc_timer); + // balance queues if needed 1st + balance_discovered_ephemerons(task_executor); + process_discovered_ephemerons_ph2(is_alive, keep_alive, complete_gc, task_executor); + } // Soft references { @@ -226,6 +234,15 @@ update_soft_ref_master_clock(); + // Ephemerons (phase2) again before Weak references. This closes the soft-reachable + // set that arrises when the value of an ephemeron with a soft-reachable key + // refers to an otherwise weaker-than-soft-reachable key of some other ephemeron. + { + GCTraceTime(Debug, gc, ref) tt("Ephemeron-SoftClosure", gc_timer); + process_discovered_ephemerons_ph2(is_alive, keep_alive, complete_gc, task_executor); + } + + // Weak references { GCTraceTime(Debug, gc, ref) tt("WeakReference", gc_timer); @@ -233,6 +250,12 @@ is_alive, keep_alive, complete_gc, task_executor); } + // Ephemerons (phase3). This clears any remaining ephemerons. + { + GCTraceTime(Debug, gc, ref) tt("Ephemeron-Clear", gc_timer); + process_discovered_ephemerons_ph3(is_alive, keep_alive, complete_gc, task_executor); + } + // Final references { GCTraceTime(Debug, gc, ref) tt("FinalReference", gc_timer); @@ -245,12 +268,6 @@ GCTraceTime(Debug, gc, ref) tt("PhantomReference", gc_timer); process_discovered_reflist(_discoveredPhantomRefs, NULL, true, is_alive, keep_alive, complete_gc, task_executor); - - // Process cleaners, but include them in phantom timing. We expect - // Cleaner references to be temporary, and don't want to deal with - // possible incompatibilities arising from making it more visible. - process_discovered_reflist(_discoveredCleanerRefs, NULL, true, - is_alive, keep_alive, complete_gc, task_executor); } // Weak global JNI references. It would make more sense (semantically) to @@ -266,8 +283,8 @@ process_phaseJNI(is_alive, keep_alive, complete_gc); } - log_debug(gc, ref)("Ref Counts: Soft: " SIZE_FORMAT " Weak: " SIZE_FORMAT " Final: " SIZE_FORMAT " Phantom: " SIZE_FORMAT, - stats.soft_count(), stats.weak_count(), stats.final_count(), stats.phantom_count()); + log_debug(gc, ref)("Ref Counts: Soft: " SIZE_FORMAT " Weak: " SIZE_FORMAT " Ephemeron: " SIZE_FORMAT " Final: " SIZE_FORMAT " Phantom: " SIZE_FORMAT, + stats.soft_count(), stats.weak_count(), stats.ephemeron_count(), stats.final_count(), stats.phantom_count()); log_develop_trace(gc, ref)("JNI Weak Reference count: " SIZE_FORMAT, count_jni_refs()); return stats; @@ -590,12 +607,153 @@ ) } -// Traverse the list and process the referents, by either -// clearing them or keeping them (and their reachable +bool +ReferenceProcessor::pp2_ephemerons_work(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { + assert(discovery_is_atomic(), "Error"); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + // Temporary list used to reverse the order of ephemerons at each pass to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in the list "preceeding" this ephemeron. + DiscoveredList reversed_list; + bool ephemerons_removed = false; + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); + oop obj = iter.obj(); + DEBUG_ONLY(oop next = java_lang_ref_Reference::next(obj);) + assert(next == NULL, "Should not discover inactive Reference"); + if (iter.is_referent_alive()) { + log_develop_trace(gc, ref)("Dropping strongly reachable reference (" INTPTR_FORMAT ": %s)", + p2i(obj), obj->klass()->internal_name()); + // The referent (key) is reachable after all. + // Remove Ephemeron object from list. + iter.remove(); + // Update the referent (key) pointer as necessary: Note that this + // should not entail any recursive marking because the + // referent must already have been traversed. + iter.make_referent_alive(); + // Update the value pointer as necessary. + HeapWord* value_addr = java_lang_ref_Ephemeron::value_addr(obj); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) value_addr); + } else { + keep_alive->do_oop((oop*) value_addr); + } + ephemerons_removed = true; + // Close the newly reachable set as soon as the value is marked to be alive + // to increase the chance other ephemeron referents (keys) are revived as + // we proceed scanning the list. + complete_gc->do_void(); + } else { + // Referent (key) is not alive (yet) so move the ephemeron to a reversed_list + // to reverse scanning in the next pass. + iter.remove(); + HeapWord* discovered_addr = java_lang_ref_Reference::discovered_addr(obj); + oop current_head = reversed_list.head(); + // The last ref must have its discovered field pointing to itself. + oop next_discovered = (current_head != NULL) ? current_head : obj; + oop_store_raw(discovered_addr, next_discovered); + reversed_list.set_head(obj); + reversed_list.inc_length(1); + } + iter.move_to_next(); + } + assert(refs_list.length() == 0, "Should be empty"); + // replace the list with reversed list + refs_list = reversed_list; + NOT_PRODUCT( + if (iter.processed() > 0) { + log_develop_trace(gc, ref)(" Dropped " SIZE_FORMAT " active Ephemerons out of " SIZE_FORMAT + " Ephemerons in discovered list " INTPTR_FORMAT, + iter.processed() - refs_list.length(), iter.processed(), + p2i(refs_list.head())); + } + ) + return ephemerons_removed; +} + +bool +ReferenceProcessor::pp2_ephemerons_work_concurrent_discovery(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { + assert(!discovery_is_atomic(), "Error"); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + // Temporary list used to reverse the order of ephemerons at each pass to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in the list "preceeding" this ephemeron. + DiscoveredList reversed_list; + bool ephemerons_removed = false; + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); + oop obj = iter.obj(); + HeapWord* next_addr = java_lang_ref_Reference::next_addr(obj); + oop next = java_lang_ref_Reference::next(obj); + if ((iter.referent() == NULL || iter.is_referent_alive() || + next != NULL)) { + assert(next->is_oop_or_null(), "Expected an oop or NULL for next field at " PTR_FORMAT, p2i(next)); + // Remove Reference object from list + iter.remove(); + // Trace the cohorts + iter.make_referent_alive(); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) next_addr); + } else { + keep_alive->do_oop((oop*) next_addr); + } + HeapWord* value_addr = java_lang_ref_Ephemeron::value_addr(obj); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) value_addr); + } else { + keep_alive->do_oop((oop*) value_addr); + } + ephemerons_removed = true; + // Close the newly reachable set as soon as the value is marked to be alive + // to increase the chance other ephemeron keys are revived as we proceed + // scanning the list + complete_gc->do_void(); + } else { + // Referent (key) is not alive (yet) so move the ephemeron to a reversed_list + // to reverse scanning in the next pass. + iter.remove(); + HeapWord* discovered_addr = java_lang_ref_Reference::discovered_addr(obj); + oop current_head = reversed_list.head(); + // The last ref must have its discovered field pointing to itself. + oop next_discovered = (current_head != NULL) ? current_head : obj; + oop_store_raw(discovered_addr, next_discovered); + reversed_list.set_head(obj); + reversed_list.inc_length(1); + } + iter.move_to_next(); + } + assert(refs_list.length() == 0, "Should be empty"); + // replace the list with reversed list + refs_list = reversed_list; + // Now close the newly reachable set at least once after the whole list has + // been scanned even if there were no ephemerons + if (!ephemerons_removed) { + complete_gc->do_void(); + } + NOT_PRODUCT( + if (iter.processed() > 0) { + log_develop_trace(gc, ref)(" Dropped " SIZE_FORMAT " active Ephemerons out of " SIZE_FORMAT + " Ephemerons in discovered list " INTPTR_FORMAT, + iter.processed() - refs_list.length(), iter.processed(), + p2i(refs_list.head())); + } + ) + return ephemerons_removed; +} + +// Traverse the list and process the referents (and values in case of Ephemerons), +// by either clearing them or keeping them (and their reachable // closure) alive. void ReferenceProcessor::process_phase3(DiscoveredList& refs_list, bool clear_referent, + bool has_ephemerons, BoolObjectClosure* is_alive, OopClosure* keep_alive, VoidClosure* complete_gc) { @@ -610,6 +768,12 @@ // keep the referent around iter.make_referent_alive(); } + if (has_ephemerons) { + assert(clear_referent, "Ephemerons should always be cleared"); + HeapWord* value_addr = java_lang_ref_Ephemeron::value_addr(iter.obj()); + // NULL out value pointer + oop_store_raw(value_addr, NULL); + } log_develop_trace(gc, ref)("Adding %sreference (" INTPTR_FORMAT ": %s) as pending", clear_referent ? "cleared " : "", p2i(iter.obj()), iter.obj()->klass()->internal_name()); assert(iter.obj()->is_oop(UseConcMarkSweepGC), "Adding a bad reference"); @@ -668,16 +832,25 @@ public: RefProcPhase2Task(ReferenceProcessor& ref_processor, DiscoveredList refs_lists[], + bool has_ephemerons, bool marks_oops_alive) - : ProcessTask(ref_processor, refs_lists, marks_oops_alive) + : ProcessTask(ref_processor, refs_lists, marks_oops_alive), + _has_ephemerons(has_ephemerons), ephemerons_removed(false) { } virtual void work(unsigned int i, BoolObjectClosure& is_alive, OopClosure& keep_alive, VoidClosure& complete_gc) { - _ref_processor.process_phase2(_refs_lists[i], - &is_alive, &keep_alive, &complete_gc); + bool r = _ref_processor.process_phase2(_refs_lists[i], _has_ephemerons, + &is_alive, &keep_alive, &complete_gc); + if (r) { + ephemerons_removed = true; + } } +private: + bool _has_ephemerons; +public: + bool ephemerons_removed; }; class RefProcPhase3Task: public AbstractRefProcTaskExecutor::ProcessTask { @@ -685,9 +858,11 @@ RefProcPhase3Task(ReferenceProcessor& ref_processor, DiscoveredList refs_lists[], bool clear_referent, + bool has_ephemerons, bool marks_oops_alive) : ProcessTask(ref_processor, refs_lists, marks_oops_alive), - _clear_referent(clear_referent) + _clear_referent(clear_referent), + _has_ephemerons(has_ephemerons) { } virtual void work(unsigned int i, BoolObjectClosure& is_alive, OopClosure& keep_alive, @@ -698,11 +873,12 @@ // Thread* thr = Thread::current(); // int refs_list_index = ((WorkerThread*)thr)->id(); // _ref_processor.process_phase3(_refs_lists[refs_list_index], _clear_referent, - _ref_processor.process_phase3(_refs_lists[i], _clear_referent, + _ref_processor.process_phase3(_refs_lists[i], _clear_referent, _has_ephemerons, &is_alive, &keep_alive, &complete_gc); } private: bool _clear_referent; + bool _has_ephemerons; }; #ifndef PRODUCT @@ -805,9 +981,9 @@ void ReferenceProcessor::balance_all_queues() { balance_queues(_discoveredSoftRefs); balance_queues(_discoveredWeakRefs); + balance_queues(_discoveredEphemerons); balance_queues(_discoveredFinalRefs); balance_queues(_discoveredPhantomRefs); - balance_queues(_discoveredCleanerRefs); } void ReferenceProcessor::process_discovered_reflist( @@ -856,22 +1032,117 @@ // Phase 2: // . Traverse the list and remove any refs whose referents are alive. if (mt_processing) { - RefProcPhase2Task phase2(*this, refs_lists, !discovery_is_atomic() /*marks_oops_alive*/); + RefProcPhase2Task phase2(*this, refs_lists, + false /*has_ephemerons*/, + !discovery_is_atomic() /*marks_oops_alive*/); task_executor->execute(phase2); } else { for (uint i = 0; i < _max_num_q; i++) { - process_phase2(refs_lists[i], is_alive, keep_alive, complete_gc); + process_phase2(refs_lists[i], + false /*has_ephemerons*/, + is_alive, keep_alive, complete_gc); } } // Phase 3: // . Traverse the list and process referents as appropriate. if (mt_processing) { - RefProcPhase3Task phase3(*this, refs_lists, clear_referent, true /*marks_oops_alive*/); + RefProcPhase3Task phase3(*this, refs_lists, clear_referent, + false /*has_ephemerons*/, + true /*marks_oops_alive*/); task_executor->execute(phase3); } else { for (uint i = 0; i < _max_num_q; i++) { process_phase3(refs_lists[i], clear_referent, + false /*has_ephemerons*/, + is_alive, keep_alive, complete_gc); + } + } +} + +// Balance ephemerons queues if needed +void ReferenceProcessor::balance_discovered_ephemerons( + AbstractRefProcTaskExecutor* task_executor) { + + bool mt_processing = task_executor != NULL && _processing_is_mt; + // If discovery used MT and a dynamic number of GC threads, then + // the queues must be balanced for correctness if fewer than the + // maximum number of queues were used. The number of queue used + // during discovery may be different than the number to be used + // for processing so don't depend of _num_q < _max_num_q as part + // of the test. + bool must_balance = _discovery_is_mt; + + if ((mt_processing && ParallelRefProcBalancingEnabled) || + must_balance) { + balance_queues(_discoveredEphemerons); + } +} + +// Process ephemerons, phase2 +void ReferenceProcessor::process_discovered_ephemerons_ph2( + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + AbstractRefProcTaskExecutor* task_executor) { + + // Traverse the _discoveredEphemerons lists and for those ephemerons whose keys + // are alive, remove them from the list, mark their values alive and close the + // reachable set. Iterate until lists become stable while reversing the + // direction of scanning in each pass. + bool mt_processing = task_executor != NULL && _processing_is_mt; + bool ephemerons_removed; + bool forward_scan = true; + do { + if (mt_processing) { + RefProcPhase2Task phase2(*this, _discoveredEphemerons, + true /*has_ephemerons*/, + !discovery_is_atomic() /*marks_oops_alive*/); + task_executor->execute(phase2); + ephemerons_removed = phase2.ephemerons_removed; + } else { + ephemerons_removed = false; + // alternate direction of selecting individual lists for scanning to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in "previous" lists... + if (forward_scan) { + for (uint i = 0; i < _max_num_q; i++) { + ephemerons_removed |= process_phase2(_discoveredEphemerons[i], + true /*has_ephemerons*/, + is_alive, keep_alive, complete_gc); + } + } else { + for (uint i = _max_num_q - 1; i < (uint) - 1; i--) { + ephemerons_removed |= process_phase2(_discoveredEphemerons[i], + true /*has_ephemerons*/, + is_alive, keep_alive, complete_gc); + } + } + forward_scan = !forward_scan; + } + } while (ephemerons_removed); +} + +// Process ephemerons, phase3 +void ReferenceProcessor::process_discovered_ephemerons_ph3( + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + AbstractRefProcTaskExecutor* task_executor) { + + // Traverse the _discoveredEphemerons lists and clear ephemerons. + bool mt_processing = task_executor != NULL && _processing_is_mt; + if (mt_processing) { + RefProcPhase3Task phase3(*this, _discoveredEphemerons, + true /*clear_referent*/, + true /*has_ephemerons*/, + true /*marks_oops_alive*/); + task_executor->execute(phase3); + } else { + for (uint i = 0; i < _max_num_q; i++) { + process_phase3(_discoveredEphemerons[i], + true /*clear_referent*/, + true /*has_ephemerons*/, is_alive, keep_alive, complete_gc); } } @@ -906,15 +1177,15 @@ case REF_WEAK: list = &_discoveredWeakRefs[id]; break; + case REF_EPHEMERON: + list = &_discoveredEphemerons[id]; + break; case REF_FINAL: list = &_discoveredFinalRefs[id]; break; case REF_PHANTOM: list = &_discoveredPhantomRefs[id]; break; - case REF_CLEANER: - list = &_discoveredCleanerRefs[id]; - break; case REF_NONE: // we should not reach here if we are an InstanceRefKlass default: @@ -1116,6 +1387,39 @@ YieldClosure* yield, GCTimer* gc_timer) { + // Ephemerons - iterate until the lists become stable + { + GCTraceTime(Debug, gc, ref) tt("Preclean Ephemerons", gc_timer); + bool ephemerons_removed; + bool forward_scan = true; + do { + ephemerons_removed = false; + // alternate direction of selecting individual lists for scanning to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in "previous" lists... + if (forward_scan) { + for (uint i = 0; i < _max_num_q; i++) { + if (yield->should_return()) { + return; + } + ephemerons_removed |= + preclean_discovered_ephemerons_reflist(_discoveredEphemerons[i], is_alive, + keep_alive, complete_gc, yield); + } + } else { + for (uint i = _max_num_q - 1; i < (uint) - 1; i--) { + if (yield->should_return()) { + return; + } + ephemerons_removed |= + preclean_discovered_ephemerons_reflist(_discoveredEphemerons[i], is_alive, + keep_alive, complete_gc, yield); + } + } + forward_scan = !forward_scan; + } while (ephemerons_removed); + } + // Soft references { GCTraceTime(Debug, gc, ref) tm("Preclean SoftReferences", gc_timer); @@ -1162,17 +1466,6 @@ preclean_discovered_reflist(_discoveredPhantomRefs[i], is_alive, keep_alive, complete_gc, yield); } - - // Cleaner references. Included in timing for phantom references. We - // expect Cleaner references to be temporary, and don't want to deal with - // possible incompatibilities arising from making it more visible. - for (uint i = 0; i < _max_num_q; i++) { - if (yield->should_return()) { - return; - } - preclean_discovered_reflist(_discoveredCleanerRefs[i], is_alive, - keep_alive, complete_gc, yield); - } } } @@ -1227,6 +1520,83 @@ } ) } +// The same as above, but specialized for ephemerons and returns true if any +// ephemerons were removed from ref list. +bool +ReferenceProcessor::preclean_discovered_ephemerons_reflist(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + YieldClosure* yield) { + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + // Temporary list used to reverse the order of ephemerons at each pass to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in the list "preceeding" this ephemeron. + DiscoveredList reversed_list; + bool ephemerons_removed = false; + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); + oop obj = iter.obj(); + oop next = java_lang_ref_Reference::next(obj); + if (iter.referent() == NULL || iter.is_referent_alive() || + next != NULL) { + // The referent has been cleared, or is alive, or the Reference is not + // active; we need to trace and mark its cohort. + log_develop_trace(gc, ref)("Precleaning Ephemeron (" INTPTR_FORMAT ": %s)", + p2i(obj), obj->klass()->internal_name()); + // Remove Reference object from list + iter.remove(); + // Keep alive its cohort. + iter.make_referent_alive(); + if (UseCompressedOops) { + narrowOop* next_addr = (narrowOop*) java_lang_ref_Reference::next_addr(obj); + keep_alive->do_oop(next_addr); + } else { + oop* next_addr = (oop*) java_lang_ref_Reference::next_addr(obj); + keep_alive->do_oop(next_addr); + } + HeapWord* value_addr = java_lang_ref_Ephemeron::value_addr(obj); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) value_addr); + } else { + keep_alive->do_oop((oop*) value_addr); + } + ephemerons_removed = true; + // Close the newly reachable set as soon as the value is marked to be alive + // to increase the chance other ephemeron referents (keys) are revived as + // we proceed scanning the list. + complete_gc->do_void(); + } else { + // Referent (key) is not alive (yet) so move the ephemeron to a reversed_list + // to reverse scanning in the next pass. + iter.remove(); + HeapWord* discovered_addr = java_lang_ref_Reference::discovered_addr(obj); + oop current_head = reversed_list.head(); + // The last ref must have its discovered field pointing to itself. + oop next_discovered = (current_head != NULL) ? current_head : obj; + oop_store_raw(discovered_addr, next_discovered); + reversed_list.set_head(obj); + reversed_list.inc_length(1); + } + iter.move_to_next(); + } + assert(refs_list.length() == 0, "Should be empty"); + // replace the list with reversed list + refs_list = reversed_list; + // Close the reachable set even if no ephemeron was removed from list + if (!ephemerons_removed) { + complete_gc->do_void(); + } + NOT_PRODUCT( + if (iter.processed() > 0) { + log_develop_trace(gc, ref)(" Dropped " SIZE_FORMAT " Ephemerons out of " SIZE_FORMAT + " Ephemerons in discovered list " INTPTR_FORMAT, + iter.processed() - refs_list.length(), iter.processed(), + p2i(refs_list.head())); + } + ) + return ephemerons_removed; +} const char* ReferenceProcessor::list_name(uint i) { assert(i <= _max_num_q * number_of_subclasses_of_ref(), @@ -1236,9 +1606,9 @@ switch (j) { case 0: return "SoftRef"; case 1: return "WeakRef"; - case 2: return "FinalRef"; - case 3: return "PhantomRef"; - case 4: return "CleanerRef"; + case 2: return "Ephemeron"; + case 3: return "FinalRef"; + case 4: return "PhantomRef"; } ShouldNotReachHere(); return NULL;