--- old/src/share/vm/gc/shared/referenceProcessor.cpp 2016-01-17 16:49:49.020582555 +0100 +++ new/src/share/vm/gc/shared/referenceProcessor.cpp 2016-01-17 16:49:48.973583366 +0100 @@ -116,9 +116,9 @@ } _discoveredSoftRefs = &_discovered_refs[0]; _discoveredWeakRefs = &_discoveredSoftRefs[_max_num_q]; - _discoveredFinalRefs = &_discoveredWeakRefs[_max_num_q]; + _discoveredEphemerons = &_discoveredWeakRefs[_max_num_q]; + _discoveredFinalRefs = &_discoveredEphemerons[_max_num_q]; _discoveredPhantomRefs = &_discoveredFinalRefs[_max_num_q]; - _discoveredCleanerRefs = &_discoveredPhantomRefs[_max_num_q]; // Initialize all entries to NULL for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { @@ -224,19 +224,17 @@ bool trace_time = PrintGCDetails && PrintReferenceGC; - // Include cleaners in phantom statistics. We expect Cleaner - // references to be temporary, and don't want to deal with - // possible incompatibilities arising from making it more visible. + // Include ephemerons in weak reference statistics for now ReferenceProcessorStats stats( total_count(_discoveredSoftRefs), - total_count(_discoveredWeakRefs), + total_count(_discoveredWeakRefs) + total_count(_discoveredEphemerons), total_count(_discoveredFinalRefs), - total_count(_discoveredPhantomRefs) + total_count(_discoveredCleanerRefs)); + total_count(_discoveredPhantomRefs)); // Soft references { GCRefTraceTime tt("SoftReference", trace_time, gc_timer, stats.soft_count()); - process_discovered_reflist(_discoveredSoftRefs, _current_soft_ref_policy, true, + process_discovered_reflist(_discoveredSoftRefs, _current_soft_ref_policy, true, false, is_alive, keep_alive, complete_gc, task_executor); } @@ -245,28 +243,26 @@ // Weak references { GCRefTraceTime tt("WeakReference", trace_time, gc_timer, stats.weak_count()); - process_discovered_reflist(_discoveredWeakRefs, NULL, true, + process_discovered_reflist(_discoveredWeakRefs, NULL, true, false, + is_alive, keep_alive, complete_gc, task_executor); + + // Process ephemerons and include them in weak reference timing for now. + process_discovered_reflist(_discoveredEphemerons, NULL, true, true, is_alive, keep_alive, complete_gc, task_executor); } // Final references { GCRefTraceTime tt("FinalReference", trace_time, gc_timer, stats.final_count()); - process_discovered_reflist(_discoveredFinalRefs, NULL, false, + process_discovered_reflist(_discoveredFinalRefs, NULL, false, false, is_alive, keep_alive, complete_gc, task_executor); } // Phantom references { GCRefTraceTime tt("PhantomReference", trace_time, gc_timer, stats.phantom_count()); - process_discovered_reflist(_discoveredPhantomRefs, NULL, false, + process_discovered_reflist(_discoveredPhantomRefs, NULL, true, false, is_alive, keep_alive, complete_gc, task_executor); - - // Process cleaners, but include them in phantom timing. We expect - // Cleaner references to be temporary, and don't want to deal with - // possible incompatibilities arising from making it more visible. - process_discovered_reflist(_discoveredCleanerRefs, NULL, true, - is_alive, keep_alive, complete_gc, task_executor); } // Weak global JNI references. It would make more sense (semantically) to @@ -618,12 +614,155 @@ ) } -// Traverse the list and process the referents, by either -// clearing them or keeping them (and their reachable +bool +ReferenceProcessor::pp2_ephemerons_work(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { + assert(discovery_is_atomic(), "Error"); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + // Temporary list used to reverse the order of ephemerons at each pass to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in the list "preceeding" this ephemeron. + DiscoveredList reversed_list; + bool ephemerons_removed = false; + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); + oop obj = iter.obj(); + DEBUG_ONLY(oop next = java_lang_ref_Reference::next(obj);) + assert(next == NULL, "Should not discover inactive Reference"); + if (iter.is_referent_alive()) { + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Dropping strongly reachable reference (" INTPTR_FORMAT ": %s)", + p2i(obj), obj->klass()->internal_name()); + } + // The referent (key) is reachable after all. + // Remove Ephemeron object from list. + iter.remove(); + // Update the referent (key) pointer as necessary: Note that this + // should not entail any recursive marking because the + // referent must already have been traversed. + iter.make_referent_alive(); + // Update the value pointer as necessary. + HeapWord* value_addr = java_lang_ref_Ephemeron::value_addr(obj); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) value_addr); + } else { + keep_alive->do_oop((oop*) value_addr); + } + ephemerons_removed = true; + // Close the newly reachable set as soon as the value is marked to be alive + // to increase the chance other ephemeron referents (keys) are revived as + // we proceed scanning the list. + complete_gc->do_void(); + } else { + // Referent (key) is not alive (yet) so move the ephemeron to a reversed_list + // to reverse scanning in the next pass. + iter.remove(); + HeapWord* discovered_addr = java_lang_ref_Reference::discovered_addr(obj); + oop current_head = reversed_list.head(); + // The last ref must have its discovered field pointing to itself. + oop next_discovered = (current_head != NULL) ? current_head : obj; + oop_store_raw(discovered_addr, next_discovered); + reversed_list.set_head(obj); + reversed_list.inc_length(1); + } + iter.move_to_next(); + } + assert(refs_list.length() == 0, "Should be empty"); + // replace the list with reversed list + refs_list = reversed_list; + NOT_PRODUCT( + if (PrintGCDetails && TraceReferenceGC && (iter.processed() > 0)) { + gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " active Ephemerons out of " SIZE_FORMAT + " Ephemerons in discovered list " INTPTR_FORMAT, + iter.processed() - refs_list.length(), iter.processed(), + p2i(refs_list.head())); + } + ) + return ephemerons_removed; +} + +bool +ReferenceProcessor::pp2_ephemerons_work_concurrent_discovery(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { + assert(!discovery_is_atomic(), "Error"); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + // Temporary list used to reverse the order of ephemerons at each pass to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in the list "preceeding" this ephemeron. + DiscoveredList reversed_list; + bool ephemerons_removed = false; + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); + oop obj = iter.obj(); + HeapWord* next_addr = java_lang_ref_Reference::next_addr(obj); + oop next = java_lang_ref_Reference::next(obj); + if ((iter.referent() == NULL || iter.is_referent_alive() || + next != NULL)) { + assert(next->is_oop_or_null(), "Expected an oop or NULL for next field at " PTR_FORMAT, p2i(next)); + // Remove Reference object from list + iter.remove(); + // Trace the cohorts + iter.make_referent_alive(); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) next_addr); + } else { + keep_alive->do_oop((oop*) next_addr); + } + HeapWord* value_addr = java_lang_ref_Ephemeron::value_addr(obj); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) value_addr); + } else { + keep_alive->do_oop((oop*) value_addr); + } + ephemerons_removed = true; + // Close the newly reachable set as soon as the value is marked to be alive + // to increase the chance other ephemeron keys are revived as we proceed + // scanning the list + complete_gc->do_void(); + } else { + // Referent (key) is not alive (yet) so move the ephemeron to a reversed_list + // to reverse scanning in the next pass. + iter.remove(); + HeapWord* discovered_addr = java_lang_ref_Reference::discovered_addr(obj); + oop current_head = reversed_list.head(); + // The last ref must have its discovered field pointing to itself. + oop next_discovered = (current_head != NULL) ? current_head : obj; + oop_store_raw(discovered_addr, next_discovered); + reversed_list.set_head(obj); + reversed_list.inc_length(1); + } + iter.move_to_next(); + } + assert(refs_list.length() == 0, "Should be empty"); + // replace the list with reversed list + refs_list = reversed_list; + // Now close the newly reachable set at least once after the whole list has + // been scanned even if there were no ephemerons + if (!ephemerons_removed) { + complete_gc->do_void(); + } + NOT_PRODUCT( + if (PrintGCDetails && TraceReferenceGC && (iter.processed() > 0)) { + gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " active Ephemerons out of " SIZE_FORMAT + " Ephemerons in discovered list " INTPTR_FORMAT, + iter.processed() - refs_list.length(), iter.processed(), + p2i(refs_list.head())); + } + ) + return ephemerons_removed; +} + +// Traverse the list and process the referents (and values in case of Ephemerons), +// by either clearing them or keeping them (and their reachable // closure) alive. void ReferenceProcessor::process_phase3(DiscoveredList& refs_list, bool clear_referent, + bool has_ephemerons, BoolObjectClosure* is_alive, OopClosure* keep_alive, VoidClosure* complete_gc) { @@ -638,6 +777,20 @@ // keep the referent around iter.make_referent_alive(); } + if (has_ephemerons) { + HeapWord* value_addr = java_lang_ref_Ephemeron::value_addr(iter.obj()); + if (clear_referent) { + // NULL out value pointer + oop_store_raw(value_addr, NULL); + } else { + // keep the value around + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) value_addr); + } else { + keep_alive->do_oop((oop*) value_addr); + } + } + } if (TraceReferenceGC) { gclog_or_tty->print_cr("Adding %sreference (" INTPTR_FORMAT ": %s) as pending", clear_referent ? "cleared " : "", @@ -699,16 +852,25 @@ public: RefProcPhase2Task(ReferenceProcessor& ref_processor, DiscoveredList refs_lists[], + bool has_ephemerons, bool marks_oops_alive) - : ProcessTask(ref_processor, refs_lists, marks_oops_alive) + : ProcessTask(ref_processor, refs_lists, marks_oops_alive), + _has_ephemerons(has_ephemerons), ephemerons_removed(false) { } virtual void work(unsigned int i, BoolObjectClosure& is_alive, OopClosure& keep_alive, VoidClosure& complete_gc) { - _ref_processor.process_phase2(_refs_lists[i], - &is_alive, &keep_alive, &complete_gc); + bool r = _ref_processor.process_phase2(_refs_lists[i], _has_ephemerons, + &is_alive, &keep_alive, &complete_gc); + if (r) { + ephemerons_removed = true; + } } +private: + bool _has_ephemerons; +public: + bool ephemerons_removed; }; class RefProcPhase3Task: public AbstractRefProcTaskExecutor::ProcessTask { @@ -716,9 +878,11 @@ RefProcPhase3Task(ReferenceProcessor& ref_processor, DiscoveredList refs_lists[], bool clear_referent, + bool has_ephemerons, bool marks_oops_alive) : ProcessTask(ref_processor, refs_lists, marks_oops_alive), - _clear_referent(clear_referent) + _clear_referent(clear_referent), + _has_ephemerons(has_ephemerons) { } virtual void work(unsigned int i, BoolObjectClosure& is_alive, OopClosure& keep_alive, @@ -729,11 +893,12 @@ // Thread* thr = Thread::current(); // int refs_list_index = ((WorkerThread*)thr)->id(); // _ref_processor.process_phase3(_refs_lists[refs_list_index], _clear_referent, - _ref_processor.process_phase3(_refs_lists[i], _clear_referent, + _ref_processor.process_phase3(_refs_lists[i], _clear_referent, _has_ephemerons, &is_alive, &keep_alive, &complete_gc); } private: bool _clear_referent; + bool _has_ephemerons; }; // Balances reference queues. @@ -835,20 +1000,24 @@ void ReferenceProcessor::balance_all_queues() { balance_queues(_discoveredSoftRefs); balance_queues(_discoveredWeakRefs); + balance_queues(_discoveredEphemerons); balance_queues(_discoveredFinalRefs); balance_queues(_discoveredPhantomRefs); - balance_queues(_discoveredCleanerRefs); } void ReferenceProcessor::process_discovered_reflist( DiscoveredList refs_lists[], ReferencePolicy* policy, bool clear_referent, + bool has_ephemerons, BoolObjectClosure* is_alive, OopClosure* keep_alive, VoidClosure* complete_gc, - AbstractRefProcTaskExecutor* task_executor) -{ + AbstractRefProcTaskExecutor* task_executor) { + + assert(!has_ephemerons || clear_referent, + "Ephemerons must be cleared when enqueued."); + bool mt_processing = task_executor != NULL && _processing_is_mt; // If discovery used MT and a dynamic number of GC threads, then // the queues must be balanced for correctness if fewer than the @@ -869,6 +1038,7 @@ // policy reasons. Keep alive the transitive closure of all // such referents. if (policy != NULL) { + assert(!has_ephemerons, "Soft refs lists can't have ephemerons."); if (mt_processing) { RefProcPhase1Task phase1(*this, refs_lists, policy, true /*marks_oops_alive*/); task_executor->execute(phase1); @@ -885,23 +1055,42 @@ // Phase 2: // . Traverse the list and remove any refs whose referents are alive. - if (mt_processing) { - RefProcPhase2Task phase2(*this, refs_lists, !discovery_is_atomic() /*marks_oops_alive*/); - task_executor->execute(phase2); - } else { - for (uint i = 0; i < _max_num_q; i++) { - process_phase2(refs_lists[i], is_alive, keep_alive, complete_gc); - } - } - + // Ephemerons need iterative phase 2 processing until the lists become stable, + // since reviving of any ephemeron's value + closing the reachable set + // can revive the referent (key) of any other ephemeron... + bool ephemerons_removed; + bool forward_scan = true; + do { + if (mt_processing) { + RefProcPhase2Task phase2(*this, refs_lists, has_ephemerons, !discovery_is_atomic() /*marks_oops_alive*/); + task_executor->execute(phase2); + ephemerons_removed = phase2.ephemerons_removed; + } else { + ephemerons_removed = false; + // alternate direction of selecting individual lists for scanning to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in "previous" lists... + if (forward_scan) { + for (uint i = 0; i < _max_num_q; i++) { + ephemerons_removed |= process_phase2(refs_lists[i], has_ephemerons, is_alive, keep_alive, complete_gc); + } + } else { + for (uint i = _max_num_q-1; i < (uint)-1; i--) { + ephemerons_removed |= process_phase2(refs_lists[i], has_ephemerons, is_alive, keep_alive, complete_gc); + } + } + forward_scan = !forward_scan; + } + } while (ephemerons_removed); + // Phase 3: // . Traverse the list and process referents as appropriate. if (mt_processing) { - RefProcPhase3Task phase3(*this, refs_lists, clear_referent, true /*marks_oops_alive*/); + RefProcPhase3Task phase3(*this, refs_lists, clear_referent, has_ephemerons, true /*marks_oops_alive*/); task_executor->execute(phase3); } else { for (uint i = 0; i < _max_num_q; i++) { - process_phase3(refs_lists[i], clear_referent, + process_phase3(refs_lists[i], clear_referent, has_ephemerons, is_alive, keep_alive, complete_gc); } } @@ -936,15 +1125,15 @@ case REF_WEAK: list = &_discoveredWeakRefs[id]; break; + case REF_EPHEMERON: + list = &_discoveredEphemerons[id]; + break; case REF_FINAL: list = &_discoveredFinalRefs[id]; break; case REF_PHANTOM: list = &_discoveredPhantomRefs[id]; break; - case REF_CLEANER: - list = &_discoveredCleanerRefs[id]; - break; case REF_NONE: // we should not reach here if we are an InstanceRefKlass default: @@ -1181,6 +1370,35 @@ preclean_discovered_reflist(_discoveredWeakRefs[i], is_alive, keep_alive, complete_gc, yield); } + + // Ephemerons need iterative precleaning until the ref lists become stable. + // Included in timing for weak references. + bool ephemerons_removed; + bool forward_scan = true; + do { + ephemerons_removed = false; + // alternate direction of selecting individual lists for scanning to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in "previous" lists... + if (forward_scan) { + for (uint i = 0; i < _max_num_q; i++) { + if (yield->should_return()) { + return; + } + ephemerons_removed |= preclean_discovered_ephemerons_reflist( + _discoveredEphemerons[i], is_alive, keep_alive, complete_gc, yield); + } + } else { + for (uint i = _max_num_q-1; i < (uint)-1; i--) { + if (yield->should_return()) { + return; + } + ephemerons_removed |= preclean_discovered_ephemerons_reflist( + _discoveredEphemerons[i], is_alive, keep_alive, complete_gc, yield); + } + } + forward_scan = !forward_scan; + } while (ephemerons_removed); } // Final references @@ -1207,17 +1425,6 @@ preclean_discovered_reflist(_discoveredPhantomRefs[i], is_alive, keep_alive, complete_gc, yield); } - - // Cleaner references. Included in timing for phantom references. We - // expect Cleaner references to be temporary, and don't want to deal with - // possible incompatibilities arising from making it more visible. - for (uint i = 0; i < _max_num_q; i++) { - if (yield->should_return()) { - return; - } - preclean_discovered_reflist(_discoveredCleanerRefs[i], is_alive, - keep_alive, complete_gc, yield); - } } } @@ -1275,6 +1482,85 @@ } ) } +// The same as above, but specialized for ephemerons and returns true if any +// ephemerons were removed from ref list. +bool +ReferenceProcessor::preclean_discovered_ephemerons_reflist(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + YieldClosure* yield) { + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + // Temporary list used to reverse the order of ephemerons at each pass to avoid + // pathological cases where majority of revived ephemeron values point + // to ephemeron keys in the list "preceeding" this ephemeron. + DiscoveredList reversed_list; + bool ephemerons_removed = false; + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); + oop obj = iter.obj(); + oop next = java_lang_ref_Reference::next(obj); + if (iter.referent() == NULL || iter.is_referent_alive() || + next != NULL) { + // The referent has been cleared, or is alive, or the Reference is not + // active; we need to trace and mark its cohort. + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Precleaning Ephemeron (" INTPTR_FORMAT ": %s)", + p2i(obj), obj->klass()->internal_name()); + } + // Remove Reference object from list + iter.remove(); + // Keep alive its cohort. + iter.make_referent_alive(); + if (UseCompressedOops) { + narrowOop* next_addr = (narrowOop*) java_lang_ref_Reference::next_addr(obj); + keep_alive->do_oop(next_addr); + } else { + oop* next_addr = (oop*) java_lang_ref_Reference::next_addr(obj); + keep_alive->do_oop(next_addr); + } + HeapWord* value_addr = java_lang_ref_Ephemeron::value_addr(obj); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*) value_addr); + } else { + keep_alive->do_oop((oop*) value_addr); + } + ephemerons_removed = true; + // Close the newly reachable set as soon as the value is marked to be alive + // to increase the chance other ephemeron referents (keys) are revived as + // we proceed scanning the list. + complete_gc->do_void(); + } else { + // Referent (key) is not alive (yet) so move the ephemeron to a reversed_list + // to reverse scanning in the next pass. + iter.remove(); + HeapWord* discovered_addr = java_lang_ref_Reference::discovered_addr(obj); + oop current_head = reversed_list.head(); + // The last ref must have its discovered field pointing to itself. + oop next_discovered = (current_head != NULL) ? current_head : obj; + oop_store_raw(discovered_addr, next_discovered); + reversed_list.set_head(obj); + reversed_list.inc_length(1); + } + iter.move_to_next(); + } + assert(refs_list.length() == 0, "Should be empty"); + // replace the list with reversed list + refs_list = reversed_list; + // Close the reachable set even if no ephemeron was removed from list + if (!ephemerons_removed) { + complete_gc->do_void(); + } + NOT_PRODUCT( + if (PrintGCDetails && PrintReferenceGC && (iter.processed() > 0)) { + gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " Ephemerons out of " SIZE_FORMAT + " Ephemerons in discovered list " INTPTR_FORMAT, + iter.processed() - refs_list.length(), iter.processed(), + p2i(refs_list.head())); + } + ) + return ephemerons_removed; +} const char* ReferenceProcessor::list_name(uint i) { assert(i <= _max_num_q * number_of_subclasses_of_ref(), @@ -1284,9 +1570,9 @@ switch (j) { case 0: return "SoftRef"; case 1: return "WeakRef"; - case 2: return "FinalRef"; - case 3: return "PhantomRef"; - case 4: return "CleanerRef"; + case 2: return "Ephemeron"; + case 3: return "FinalRef"; + case 4: return "PhantomRef"; } ShouldNotReachHere(); return NULL;