src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
Print this page
rev 2585 : [mq]: g1-reference-processing
@@ -1260,13 +1260,16 @@
// This mode is disabled in
// instanceRefKlass::process_discovered_references if the
// generation does some collection work, or
// instanceRefKlass::enqueue_discovered_references if the
// generation returns without doing any work.
- ref_processor()->disable_discovery();
- ref_processor()->abandon_partial_discovery();
- ref_processor()->verify_no_references_recorded();
+
+ // Disable discovery and empty the discovered lists
+ // for the CM ref processor.
+ ref_processor_cm()->disable_discovery();
+ ref_processor_cm()->abandon_partial_discovery();
+ ref_processor_cm()->verify_no_references_recorded();
// Abandon current iterations of concurrent marking and concurrent
// refinement, if any are in progress.
concurrent_mark()->abort();
@@ -1292,35 +1295,40 @@
if (g1_policy()->in_young_gc_mode()) {
empty_young_list();
g1_policy()->set_full_young_gcs(true);
}
- // See the comment in G1CollectedHeap::ref_processing_init() about
+ // See the comments in g1CollectedHeap.hpp and
+ // G1CollectedHeap::ref_processing_init() about
// how reference processing currently works in G1.
- // Temporarily make reference _discovery_ single threaded (non-MT).
- ReferenceProcessorMTDiscoveryMutator rp_disc_ser(ref_processor(), false);
+ assert(!ref_processor_stw()->discovery_enabled(), "Precondition");
+ ref_processor_stw()->verify_no_references_recorded();
+
+ // Temporarily make discovery by the STW ref processor single threaded (non-MT).
+ ReferenceProcessorMTDiscoveryMutator stw_rp_disc_ser(ref_processor_stw(), false);
- // Temporarily make refs discovery atomic
- ReferenceProcessorAtomicMutator rp_disc_atomic(ref_processor(), true);
+ // Temporarily clear the STW ref processor's _is_alive_non_header field.
+ ReferenceProcessorIsAliveMutator stw_rp_is_alive_null(ref_processor_stw(), NULL);
- // Temporarily clear _is_alive_non_header
- ReferenceProcessorIsAliveMutator rp_is_alive_null(ref_processor(), NULL);
+ ref_processor_stw()->enable_discovery();
+ ref_processor_stw()->setup_policy(do_clear_all_soft_refs);
- ref_processor()->enable_discovery();
- ref_processor()->setup_policy(do_clear_all_soft_refs);
// Do collection work
{
HandleMark hm; // Discard invalid handles created during gc
- G1MarkSweep::invoke_at_safepoint(ref_processor(), do_clear_all_soft_refs);
+ G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs);
}
+
assert(free_regions() == 0, "we should not have added any free regions");
rebuild_region_lists();
_summary_bytes_used = recalculate_used();
- ref_processor()->enqueue_discovered_references();
+ // Enqueue any discovered reference objects that have
+ // not been removed from the discovered lists.
+ ref_processor_stw()->enqueue_discovered_references();
COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
MemoryService::track_memory_usage();
@@ -1331,11 +1339,20 @@
Universe::verify(/* allow dirty */ false,
/* silent */ false,
/* option */ VerifyOption_G1UsePrevMarking);
}
- NOT_PRODUCT(ref_processor()->verify_no_references_recorded());
+
+ assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
+ ref_processor_stw()->verify_no_references_recorded();
+
+ // Note: since we've just done a full GC, concurrent
+ // marking is no longer active. Therefore we need not
+ // re-enable reference discovery for the CM ref processor.
+ // That will be done at the start of the next marking cycle.
+ assert(!ref_processor_cm()->discovery_enabled(), "Postcondition");
+ ref_processor_cm()->verify_no_references_recorded();
reset_gc_time_stamp();
// Since everything potentially moved, we will clear all remembered
// sets, and clear all cards. Later we will rebuild remebered
// sets. We will also reset the GC time stamps of the regions.
@@ -1803,12 +1820,14 @@
G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :
SharedHeap(policy_),
_g1_policy(policy_),
_dirty_card_queue_set(false),
_into_cset_dirty_card_queue_set(false),
- _is_alive_closure(this),
- _ref_processor(NULL),
+ _is_alive_closure_cm(this),
+ _is_alive_closure_stw(this),
+ _ref_processor_cm(NULL),
+ _ref_processor_stw(NULL),
_process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)),
_bot_shared(NULL),
_objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL),
_evac_failure_scan_stack(NULL) ,
_mark_in_progress(false),
@@ -2113,37 +2132,82 @@
}
void G1CollectedHeap::ref_processing_init() {
// Reference processing in G1 currently works as follows:
//
- // * There is only one reference processor instance that
- // 'spans' the entire heap. It is created by the code
- // below.
- // * Reference discovery is not enabled during an incremental
- // pause (see 6484982).
- // * Discoverered refs are not enqueued nor are they processed
- // during an incremental pause (see 6484982).
+ // * There are two reference processor instances. One is
+ // used to record and process discovered references
+ // during concurrent marking; the other is used to
+ // record and process references during STW pauses
+ // (both full and incremental).
+ // * Both ref processors need to 'span' the entire heap as
+ // the regions in the collection set may be dotted around.
+ //
+ // * For the concurrent marking ref processor:
// * Reference discovery is enabled at initial marking.
// * Reference discovery is disabled and the discovered
// references processed etc during remarking.
// * Reference discovery is MT (see below).
// * Reference discovery requires a barrier (see below).
- // * Reference processing is currently not MT (see 6608385).
- // * A full GC enables (non-MT) reference discovery and
- // processes any discovered references.
+ // * Reference processing may or may not be MT
+ // (depending on the value of ParallelRefProcEnabled
+ // and ParallelGCThreads).
+ // * A full GC disables reference discovery by the CM
+ // ref processor and abandons any entries on it's
+ // discovered lists.
+ //
+ // * For the STW processor:
+ // * Non MT discovery is enabled at the start of a full GC.
+ // * Processing and enqueueing during a full GC is non-MT.
+ // * During a full GC, references are processed after marking.
+ //
+ // * Discovery (may or may not be MT) is enabled at the start
+ // of an incremental evacuation pause.
+ // * References are processed near the end of a STW evacuation pause.
+ // * For both types of GC:
+ // * Discovery is atomic - i.e. not concurrent.
+ // * Reference discovery will not need a barrier.
SharedHeap::ref_processing_init();
MemRegion mr = reserved_region();
- _ref_processor =
+
+ // Concurrent Mark ref processor
+ _ref_processor_cm =
+ new ReferenceProcessor(mr, // span
+ ParallelRefProcEnabled && (ParallelGCThreads > 1),
+ // mt processing
+ (int) ParallelGCThreads,
+ // degree of mt processing
+ (ParallelGCThreads > 1) || (ConcGCThreads > 1),
+ // mt discovery
+ (int) MAX2(ParallelGCThreads, ConcGCThreads),
+ // degree of mt discovery
+ false,
+ // Reference discovery is not atomic
+ &_is_alive_closure_cm,
+ // is alive closure for efficiency
+ true);
+ // Setting next fields of discovered
+ // lists requires a barrier.
+
+ // STW ref processor
+ _ref_processor_stw =
new ReferenceProcessor(mr, // span
- ParallelRefProcEnabled && (ParallelGCThreads > 1), // mt processing
- (int) ParallelGCThreads, // degree of mt processing
- ParallelGCThreads > 1 || ConcGCThreads > 1, // mt discovery
- (int) MAX2(ParallelGCThreads, ConcGCThreads), // degree of mt discovery
- false, // Reference discovery is not atomic
- &_is_alive_closure, // is alive closure for efficiency
- true); // Setting next fields of discovered
+ ParallelRefProcEnabled && (ParallelGCThreads > 1),
+ // mt processing
+ MAX2((int)ParallelGCThreads, 1),
+ // degree of mt processing
+ (ParallelGCThreads > 1),
+ // mt discovery
+ MAX2((int)ParallelGCThreads, 1),
+ // degree of mt discovery
+ true,
+ // Reference discovery is atomic
+ &_is_alive_closure_stw,
+ // is alive closure for efficiency
+ false);
+ // Setting next fields of discovered
// lists requires a barrier.
}
size_t G1CollectedHeap::capacity() const {
return _g1_committed.byte_size();
@@ -3184,10 +3248,14 @@
// I'm ignoring the "fill_newgen()" call if "alloc_event_enabled"
// is set.
COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(),
"derived pointer present"));
// always_do_update_barrier = true;
+
+ // We have just completed a GC. Update the soft reference
+ // policy with the new heap occupancy
+ Universe::update_heap_info_at_gc();
}
HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size,
unsigned int gc_count_before,
bool* succeeded) {
@@ -3431,17 +3499,26 @@
}
COMPILER2_PRESENT(DerivedPointerTable::clear());
- // Please see comment in G1CollectedHeap::ref_processing_init()
- // to see how reference processing currently works in G1.
- //
- // We want to turn off ref discovery, if necessary, and turn it back on
- // on again later if we do. XXX Dubious: why is discovery disabled?
- bool was_enabled = ref_processor()->discovery_enabled();
- if (was_enabled) ref_processor()->disable_discovery();
+ // Please see comment in g1CollectedHeap.hpp and
+ // G1CollectedHeap::ref_processing_init() to see how
+ // reference processing currently works in G1.
+
+ assert(!ref_processor_stw()->discovery_enabled(), "Precondition");
+ ref_processor_stw()->verify_no_references_recorded();
+
+ // Enable discovery in the STW reference processor
+ ref_processor_stw()->enable_discovery();
+
+ {
+ // We want to temporarily turn off discovery by the
+ // CM ref processor, if necessary, and turn it back on
+ // on again later if we do. Using a scoped
+ // NoRefDiscovery object will do this.
+ NoRefDiscovery no_cm_discovery(ref_processor_cm());
// Forget the current alloc region (we might even choose it to be part
// of the collection set!).
release_mutator_alloc_region();
@@ -3621,11 +3698,15 @@
Universe::verify(/* allow dirty */ true,
/* silent */ false,
/* option */ VerifyOption_G1UsePrevMarking);
}
- if (was_enabled) ref_processor()->enable_discovery();
+ assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
+ ref_processor_stw()->verify_no_references_recorded();
+
+ // CM reference discovery will be re-enabled if necessary.
+ }
{
size_t expand_bytes = g1_policy()->expansion_amount();
if (expand_bytes > 0) {
size_t bytes_before = capacity();
@@ -3972,58 +4053,10 @@
assert(!_drain_in_progress, "Postcondition");
delete _evac_failure_scan_stack;
_evac_failure_scan_stack = NULL;
}
-
-
-// *** Sequential G1 Evacuation
-
-class G1IsAliveClosure: public BoolObjectClosure {
- G1CollectedHeap* _g1;
-public:
- G1IsAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
- void do_object(oop p) { assert(false, "Do not call."); }
- bool do_object_b(oop p) {
- // It is reachable if it is outside the collection set, or is inside
- // and forwarded.
-
-#ifdef G1_DEBUG
- gclog_or_tty->print_cr("is alive "PTR_FORMAT" in CS %d forwarded %d overall %d",
- (void*) p, _g1->obj_in_cs(p), p->is_forwarded(),
- !_g1->obj_in_cs(p) || p->is_forwarded());
-#endif // G1_DEBUG
-
- return !_g1->obj_in_cs(p) || p->is_forwarded();
- }
-};
-
-class G1KeepAliveClosure: public OopClosure {
- G1CollectedHeap* _g1;
-public:
- G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
- void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
- void do_oop( oop* p) {
- oop obj = *p;
-#ifdef G1_DEBUG
- if (PrintGC && Verbose) {
- gclog_or_tty->print_cr("keep alive *"PTR_FORMAT" = "PTR_FORMAT" "PTR_FORMAT,
- p, (void*) obj, (void*) *p);
- }
-#endif // G1_DEBUG
-
- if (_g1->obj_in_cs(obj)) {
- assert( obj->is_forwarded(), "invariant" );
- *p = obj->forwardee();
-#ifdef G1_DEBUG
- gclog_or_tty->print_cr(" in CSet: moved "PTR_FORMAT" -> "PTR_FORMAT,
- (void*) obj, (void*) *p);
-#endif // G1_DEBUG
- }
- }
-};
-
class UpdateRSetDeferred : public OopsInHeapRegionClosure {
private:
G1CollectedHeap* _g1;
DirtyCardQueue *_dcq;
CardTableModRefBS* _ct_bs;
@@ -4526,16 +4559,21 @@
}
}
#endif // ASSERT
void G1ParScanThreadState::trim_queue() {
+ assert(_evac_cl != NULL, "not set");
+ assert(_evac_failure_cl != NULL, "not set");
+ assert(_partial_scan_cl != NULL, "not set");
+
StarTask ref;
do {
// Drain the overflow stack first, so other threads can steal.
while (refs()->pop_overflow(ref)) {
deal_with_reference(ref);
}
+
while (refs()->pop_local(ref)) {
deal_with_reference(ref);
}
} while (!refs()->is_empty());
}
@@ -4651,12 +4689,11 @@
return obj;
}
template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee>
template <class T>
-void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee>
-::do_oop_work(T* p) {
+void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee>::do_oop_work(T* p) {
oop obj = oopDesc::load_decode_heap_oop(p);
assert(barrier != G1BarrierRS || obj != NULL,
"Precondition: G1BarrierRS implies obj is nonNull");
// here the null check is implicit in the cset_fast_test() test
@@ -4813,39 +4850,46 @@
_g1h->g1_policy()->record_gc_worker_start_time(i, start_time_ms);
ResourceMark rm;
HandleMark hm;
+ ReferenceProcessor* rp = _g1h->ref_processor_stw();
+
G1ParScanThreadState pss(_g1h, i);
- G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss);
- G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss);
- G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss);
+ G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss, rp);
+ G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, rp);
+ G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss, rp);
pss.set_evac_closure(&scan_evac_cl);
pss.set_evac_failure_closure(&evac_failure_cl);
pss.set_partial_scan_closure(&partial_scan_cl);
- G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss);
- G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss);
- G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss);
- G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, &pss);
-
- G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss);
- G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss);
- G1ParScanAndMarkHeapRSClosure scan_mark_heap_rs_cl(_g1h, &pss);
+ G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss, rp);
+ G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss, rp);
- OopsInHeapRegionClosure *scan_root_cl;
- OopsInHeapRegionClosure *scan_perm_cl;
+ G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss, rp);
+ G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss, rp);
+
+ OopClosure* scan_root_cl = &only_scan_root_cl;
+ OopsInHeapRegionClosure* scan_perm_cl = &only_scan_perm_cl;
if (_g1h->g1_policy()->during_initial_mark_pause()) {
+ // We also need to mark copied objects.
scan_root_cl = &scan_mark_root_cl;
scan_perm_cl = &scan_mark_perm_cl;
- } else {
- scan_root_cl = &only_scan_root_cl;
- scan_perm_cl = &only_scan_perm_cl;
}
+ // The following closure is used to scan RSets looking for reference
+ // fields that point into the collection set. The actual field iteration
+ // is performed by a FilterIntoCSClosure, whose do_oop method calls the
+ // do_oop method of the following closure.
+ // Therefore we want to record the reference processor in the
+ // FilterIntoCSClosure. To do so we record the STW reference
+ // processor into the following closure and pass it to the
+ // FilterIntoCSClosure in HeapRegionDCTOC::walk_mem_region_with_cl.
+ G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, &pss, rp);
+
pss.start_strong_roots();
_g1h->g1_process_strong_roots(/* not collecting perm */ false,
SharedHeap::SO_AllClasses,
scan_root_cl,
&push_heap_rs_cl,
@@ -4889,10 +4933,11 @@
SharedHeap::ScanningOption so,
OopClosure* scan_non_heap_roots,
OopsInHeapRegionClosure* scan_rs,
OopsInGenClosure* scan_perm,
int worker_i) {
+
// First scan the strong roots, including the perm gen.
double ext_roots_start = os::elapsedTime();
double closure_app_time_sec = 0.0;
BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots);
@@ -4907,17 +4952,18 @@
collecting_perm_gen, so,
&buf_scan_non_heap_roots,
&eager_scan_code_roots,
&buf_scan_perm);
- // Now the ref_processor roots.
+ // Now the CM ref_processor roots.
if (!_process_strong_tasks->is_task_claimed(G1H_PS_refProcessor_oops_do)) {
- // We need to treat the discovered reference lists as roots and
- // keep entries (which are added by the marking threads) on them
- // live until they can be processed at the end of marking.
- ref_processor()->weak_oops_do(&buf_scan_non_heap_roots);
- ref_processor()->oops_do(&buf_scan_non_heap_roots);
+ // We need to treat the discovered reference lists of the
+ // concurrent mark ref processor as roots and keep entries
+ // (which are added by the marking threads) on them live
+ // until they can be processed at the end of marking.
+ ref_processor_cm()->weak_oops_do(&buf_scan_non_heap_roots);
+ ref_processor_cm()->oops_do(&buf_scan_non_heap_roots);
}
// Finish up any enqueued closure apps (attributed as object copy time).
buf_scan_non_heap_roots.done();
buf_scan_perm.done();
@@ -4972,10 +5018,551 @@
}
// We do this even in the parallel case
perm_gen()->save_marks();
}
+// Weak Reference Processing support
+
+bool G1STWIsAliveClosure::do_object_b(oop p) {
+ // An object is reachable if it is outside the collection set,
+ // or is inside and copied.
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr("is alive "PTR_FORMAT" in CS %d forwarded %d overall %d",
+ (void*) p, _g1->obj_in_cs(p), p->is_forwarded(),
+ !_g1->obj_in_cs(p) || p->is_forwarded());
+#endif // G1_DEBUG
+ return !_g1->obj_in_cs(p) || p->is_forwarded();
+}
+
+// Non Copying Keep Alive closure
+class G1KeepAliveClosure: public OopClosure {
+ G1CollectedHeap* _g1;
+public:
+ G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
+ void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
+ void do_oop( oop* p) {
+ oop obj = *p;
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr("Non-copy keep alive *"PTR_FORMAT" = "PTR_FORMAT" "PTR_FORMAT,
+ p, (void*) obj, (void*) *p);
+#endif // G1_DEBUG
+
+ if (_g1->obj_in_cs(obj)) {
+ assert( obj->is_forwarded(), "invariant" );
+ *p = obj->forwardee();
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr(" in CSet: moved "PTR_FORMAT" -> "PTR_FORMAT,
+ (void*) obj, (void*) *p);
+#endif // G1_DEBUG
+ }
+ }
+};
+
+// Copying Keep Alive closure - can be called from both
+// serial and parallel code as long as different worker
+// threads utilize different G1ParScanThreadState instances
+// and different queues.
+
+class G1CopyingKeepAliveClosure: public OopClosure {
+ G1CollectedHeap* _g1h;
+ OopClosure* _copy_non_heap_obj_cl;
+ OopsInHeapRegionClosure* _copy_perm_obj_cl;
+ G1ParScanThreadState* _par_scan_state;
+
+public:
+ G1CopyingKeepAliveClosure(G1CollectedHeap* g1h,
+ OopClosure* non_heap_obj_cl,
+ OopsInHeapRegionClosure* perm_obj_cl,
+ G1ParScanThreadState* pss):
+ _g1h(g1h),
+ _copy_non_heap_obj_cl(non_heap_obj_cl),
+ _copy_perm_obj_cl(perm_obj_cl),
+ _par_scan_state(pss)
+ {}
+
+ virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+ virtual void do_oop( oop* p) { do_oop_work(p); }
+
+ template <class T> void do_oop_work(T* p) {
+ oop obj = oopDesc::load_decode_heap_oop(p);
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr("Copying keep alive *"PTR_FORMAT" = "PTR_FORMAT" "PTR_FORMAT,
+ p, (void*) obj, (void*) *p);
+#endif // G1_DEBUG
+
+ if (_g1h->obj_in_cs(obj)) {
+ // If the referent object has been forwarded (either copied
+ // to a new location or to itself in the event of an
+ // evacuation failure) then we need to update the reference
+ // field and, if both reference and referent are in the G1
+ // heap, update the RSet for the referent.
+ //
+ // If the referent has not been forwarded then we have to keep
+ // it alive by policy. Therefore we have copy the referent.
+ // If the reference field is in the G1 heap then we can push
+ // on the PSS queue. When the queue is drained (after each
+ // phase of reference processing) the object and it's followers
+ // will be copied. If the reference field is not in the G1 heap
+ // then we need to use the the non-heap or perm closures to
+ // copy the object to avoid updating the RSet.
+#ifdef G1_DEBUG
+ if (obj->is_forwarded()) {
+ gclog_or_tty->print_cr(" in CSet: forwarded "PTR_FORMAT" -> "PTR_FORMAT,
+ (void*) obj, (void*) *p);
+ }
+#endif // G1_DEBUG
+
+ if (_g1h->is_in_g1_reserved(p)) {
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr(" in CSet: pushing "PTR_FORMAT, p);
+#endif // G1_DEBUG
+ _par_scan_state->push_on_queue(p);
+ } else {
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr(" in CSet: applying closure to "PTR_FORMAT, p);
+#endif // G1_DEBUG
+ // The reference field is not in the G1 heap.
+ if (_g1h->perm_gen()->is_in(p)) {
+ _copy_perm_obj_cl->do_oop(p);
+ } else {
+ _copy_non_heap_obj_cl->do_oop(p);
+ }
+ }
+ } else {
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr(" not in CSet");
+#endif // G1_DEBUG
+ }
+ }
+};
+
+// Serial drain queue closure. Called as the 'complete_gc'
+// closure for each discovered list in some of the
+// reference processing phases.
+
+class G1STWDrainQueueClosure: public VoidClosure {
+protected:
+ G1CollectedHeap* _g1h;
+ G1ParScanThreadState* _par_scan_state;
+
+ G1ParScanThreadState* par_scan_state() { return _par_scan_state; }
+
+public:
+ G1STWDrainQueueClosure(G1CollectedHeap* g1h, G1ParScanThreadState* pss) :
+ _g1h(g1h),
+ _par_scan_state(pss)
+ { }
+
+ void do_void() {
+ G1ParScanThreadState* const pss = par_scan_state();
+ pss->trim_queue();
+ }
+};
+
+// Parallel Reference Processing closures
+
+// Implementation of AbstractRefProcTaskExecutor for parallel reference
+// processing during G1 evacuation pauses.
+
+class G1STWRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
+private:
+ G1CollectedHeap* _g1h;
+ RefToScanQueueSet* _queues;
+ WorkGang* _workers;
+ int _active_workers;
+
+public:
+ G1STWRefProcTaskExecutor(G1CollectedHeap* g1h,
+ WorkGang* workers,
+ RefToScanQueueSet *task_queues,
+ int n_workers) :
+ _g1h(g1h),
+ _queues(task_queues),
+ _workers(workers),
+ _active_workers(n_workers)
+ {
+ assert(n_workers > 0, "shouldn't call this otherwise");
+ }
+
+ // Executes the given task using concurrent marking worker threads.
+ virtual void execute(ProcessTask& task);
+ virtual void execute(EnqueueTask& task);
+};
+
+// Gang task for possibly parallel reference processing
+
+class G1STWRefProcTaskProxy: public AbstractGangTask {
+ typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
+ ProcessTask& _proc_task;
+ G1CollectedHeap* _g1h;
+ RefToScanQueueSet *_task_queues;
+ ParallelTaskTerminator* _terminator;
+
+public:
+ G1STWRefProcTaskProxy(ProcessTask& proc_task,
+ G1CollectedHeap* g1h,
+ RefToScanQueueSet *task_queues,
+ ParallelTaskTerminator* terminator) :
+ AbstractGangTask("Process reference objects in parallel"),
+ _proc_task(proc_task),
+ _g1h(g1h),
+ _task_queues(task_queues),
+ _terminator(terminator)
+ {}
+
+ virtual void work(int i) {
+ // The reference processing task executed by a single worker.
+ ResourceMark rm;
+ HandleMark hm;
+
+ G1STWIsAliveClosure is_alive(_g1h);
+
+ G1ParScanThreadState pss(_g1h, i);
+
+ G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss, NULL);
+ G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL);
+ G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss, NULL);
+
+ pss.set_evac_closure(&scan_evac_cl);
+ pss.set_evac_failure_closure(&evac_failure_cl);
+ pss.set_partial_scan_closure(&partial_scan_cl);
+
+ G1ParScanExtRootClosure only_copy_non_heap_cl(_g1h, &pss, NULL);
+ G1ParScanPermClosure only_copy_perm_cl(_g1h, &pss, NULL);
+
+ G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(_g1h, &pss, NULL);
+ G1ParScanAndMarkPermClosure copy_mark_perm_cl(_g1h, &pss, NULL);
+
+ OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl;
+ OopsInHeapRegionClosure* copy_perm_cl = &only_copy_perm_cl;
+
+ if (_g1h->g1_policy()->during_initial_mark_pause()) {
+ // We also need to mark copied objects.
+ copy_non_heap_cl = ©_mark_non_heap_cl;
+ copy_perm_cl = ©_mark_perm_cl;
+ }
+
+ // Keep alive closure.
+ G1CopyingKeepAliveClosure keep_alive(_g1h, copy_non_heap_cl, copy_perm_cl, &pss);
+
+ // Complete GC closure
+ G1ParEvacuateFollowersClosure drain_queue(_g1h, &pss, _task_queues, _terminator);
+
+ // Call the reference processing task's work routine.
+ _proc_task.work(i, is_alive, keep_alive, drain_queue);
+
+ // Note we cannot assert that the refs array is empty here as not all
+ // of the processing tasks (specifically phase2 - pp2_work) do not execute
+ // the complete_gc closure (which ordinarily would drain the queue) so
+ // the queue may not be empty.
+ }
+};
+
+// Driver routine for parallel reference processing.
+// Creates an instance of the ref processing gang
+// task and has the worker threads execute it.
+void G1STWRefProcTaskExecutor::execute(ProcessTask& proc_task) {
+ assert(_workers != NULL, "Need parallel worker threads.");
+
+ ParallelTaskTerminator terminator(_active_workers, _queues);
+ G1STWRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _queues, &terminator);
+
+ _g1h->set_par_threads(_active_workers);
+ _workers->run_task(&proc_task_proxy);
+ _g1h->set_par_threads(0);
+}
+
+// Gang task for parallel reference enqueueing.
+
+class G1STWRefEnqueueTaskProxy: public AbstractGangTask {
+ typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
+ EnqueueTask& _enq_task;
+
+public:
+ G1STWRefEnqueueTaskProxy(EnqueueTask& enq_task) :
+ AbstractGangTask("Enqueue reference objects in parallel"),
+ _enq_task(enq_task)
+ { }
+
+ virtual void work(int i) {
+ _enq_task.work(i);
+ }
+};
+
+// Driver routine for parallel reference enqueing.
+// Creates an instance of the ref enqueueing gang
+// task and has the worker threads execute it.
+
+void G1STWRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
+ assert(_workers != NULL, "Need parallel worker threads.");
+
+ G1STWRefEnqueueTaskProxy enq_task_proxy(enq_task);
+
+ _g1h->set_par_threads(_active_workers);
+ _workers->run_task(&enq_task_proxy);
+ _g1h->set_par_threads(0);
+}
+
+// End of weak reference support closures
+
+// Abstract task used to preserve (i.e. copy) any referent objects
+// that are in the collection set and are pointed to by reference
+// objects discovered by the CM ref processor.
+
+class G1ParPreserveCMReferentsTask: public AbstractGangTask {
+protected:
+ G1CollectedHeap* _g1h;
+ RefToScanQueueSet *_queues;
+ ParallelTaskTerminator _terminator;
+ int _n_workers;
+
+public:
+ G1ParPreserveCMReferentsTask(G1CollectedHeap* g1h,int workers, RefToScanQueueSet *task_queues) :
+ AbstractGangTask("ParPreserveCMReferents"),
+ _g1h(g1h),
+ _queues(task_queues),
+ _terminator(workers, _queues),
+ _n_workers(workers)
+ { }
+
+ void work(int i) {
+ ResourceMark rm;
+ HandleMark hm;
+
+ G1ParScanThreadState pss(_g1h, i);
+ G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss, NULL);
+ G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL);
+ G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss, NULL);
+
+ pss.set_evac_closure(&scan_evac_cl);
+ pss.set_evac_failure_closure(&evac_failure_cl);
+ pss.set_partial_scan_closure(&partial_scan_cl);
+
+ assert(pss.refs()->is_empty(), "both queue and overflow should be empty");
+
+ // Copying keep alive closure. Applied to referent objects that need
+ // to be copied.
+
+ G1ParScanExtRootClosure only_copy_non_heap_cl(_g1h, &pss, NULL);
+ G1ParScanPermClosure only_copy_perm_cl(_g1h, &pss, NULL);
+
+ G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(_g1h, &pss, NULL);
+ G1ParScanAndMarkPermClosure copy_mark_perm_cl(_g1h, &pss, NULL);
+
+ OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl;
+ OopsInHeapRegionClosure* copy_perm_cl = &only_copy_perm_cl;
+
+ if (_g1h->g1_policy()->during_initial_mark_pause()) {
+ // We also need to mark copied objects.
+ copy_non_heap_cl = ©_mark_non_heap_cl;
+ copy_perm_cl = ©_mark_perm_cl;
+ }
+
+ // Keep alive closure.
+ G1CopyingKeepAliveClosure keep_alive(_g1h, copy_non_heap_cl, copy_perm_cl, &pss);
+
+ ReferenceProcessor* rp = _g1h->ref_processor_cm();
+
+ int limit = ReferenceProcessor::subclasses_of_ref() * rp->max_num_q();
+ int stride = MIN2(MAX2(_n_workers, 1), limit);
+
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr("_n_workers = %d", _n_workers);
+ gclog_or_tty->print_cr("limit = %d", limit);
+ gclog_or_tty->print_cr("stride = %d", stride);
+#endif // G1_DEBUG
+
+ // limit is set using max_num_q() - which was set using ParallelGCThreads.
+ // So this must be true - but assert just in case someone decides to
+ // change the worker ids.
+ assert(0 <= i && i < limit, "sanity");
+
+ // Select discovered lists [i, i+stride, i+2*stride,...,limit)
+ for (int idx = i; idx < limit; idx += stride) {
+ DiscoveredList& ref_list = rp->discovered_soft_refs()[idx];
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr("Worker %d: Preserving referents on %s list(%d) - "PTR_FORMAT,
+ i, rp->list_name(idx), idx, (void*)&ref_list);
+#endif // G1_DEBUG
+
+ oop ref = ref_list.head();
+ while (ref != ReferenceProcessor::sentinel_ref()) {
+#ifdef G1_DEBUG
+ if (_g1h->obj_in_cs(ref)) {
+ gclog_or_tty->print_cr("Boom! Worker %d: Found a reference in CS. Forwarded? %s",
+ i, (ref->is_forwarded() ? "YES" : "NO"));
+ }
+#endif // G1_DEBUG
+
+ HeapWord* referent_addr = java_lang_ref_Reference::referent_addr(ref);
+ oop referent = java_lang_ref_Reference::referent(ref);
+ if (referent != NULL) {
+ if (UseCompressedOops) {
+ keep_alive.do_oop((narrowOop*)referent_addr);
+ } else {
+ keep_alive.do_oop((oop*)referent_addr);
+ }
+ }
+
+ // get the next reference object on the discovered list.
+ ref = java_lang_ref_Reference::discovered(ref);
+ }
+ }
+
+ // Drain the queue - which may cause stealing
+ G1ParEvacuateFollowersClosure drain_queue(_g1h, &pss, _queues, &_terminator);
+ drain_queue.do_void();
+
+ // Allocation buffers were retired at the end of G1ParEvacuateFollowersClosure
+ assert(pss.refs()->is_empty(), "should be");
+ }
+};
+
+// Weak Reference processing during an evacuation pause (part 1).
+void G1CollectedHeap::process_discovered_references() {
+ double ref_proc_start = os::elapsedTime();
+
+ ReferenceProcessor* rp = _ref_processor_stw;
+ assert(rp->discovery_enabled(), "should have been enabled");
+
+ // Closure to test whether a referent is alive.
+ G1STWIsAliveClosure is_alive(this);
+
+ // Even when parallel reference processing is enabled, the processing
+ // of JNI refs is serial and performed serially by the current thread
+ // rather than by a worker. The following PSS will be used for processing
+ // JNI refs.
+
+ // Use only a single queue for this PSS.
+ G1ParScanThreadState pss(this, 0);
+
+ // We do not embed a reference processor in the copying closures
+ // while we're actually processing the discovered reference objects.
+ G1ParScanHeapEvacClosure scan_evac_cl(this, &pss, NULL);
+ G1ParScanHeapEvacFailureClosure evac_failure_cl(this, &pss, NULL);
+ G1ParScanPartialArrayClosure partial_scan_cl(this, &pss, NULL);
+
+ pss.set_evac_closure(&scan_evac_cl);
+ pss.set_evac_failure_closure(&evac_failure_cl);
+ pss.set_partial_scan_closure(&partial_scan_cl);
+
+ assert(pss.refs()->is_empty(), "pre-condition");
+
+ G1ParScanExtRootClosure only_copy_non_heap_cl(this, &pss, NULL);
+ G1ParScanPermClosure only_copy_perm_cl(this, &pss, NULL);
+
+ G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(this, &pss, NULL);
+ G1ParScanAndMarkPermClosure copy_mark_perm_cl(this, &pss, NULL);
+
+ OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl;
+ OopsInHeapRegionClosure* copy_perm_cl = &only_copy_perm_cl;
+
+ if (_g1h->g1_policy()->during_initial_mark_pause()) {
+ // We also need to mark copied objects.
+ copy_non_heap_cl = ©_mark_non_heap_cl;
+ copy_perm_cl = ©_mark_perm_cl;
+ }
+
+ // Keep alive closure.
+ G1CopyingKeepAliveClosure keep_alive(this, copy_non_heap_cl, copy_perm_cl, &pss);
+
+ // Serial Complete GC closure
+ G1STWDrainQueueClosure drain_queue(this, &pss);
+
+ // Setup the soft refs policy...
+ rp->setup_policy(false);
+
+ if (!rp->processing_is_mt()) {
+ // Serial reference processing...
+ rp->process_discovered_references(&is_alive,
+ &keep_alive,
+ &drain_queue,
+ NULL);
+ } else {
+ // Parallel reference processing
+ int active_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1);
+ assert(rp->num_q() == active_workers, "sanity");
+ assert(active_workers <= rp->max_num_q(), "sanity");
+
+ G1STWRefProcTaskExecutor par_task_executor(this, workers(), _task_queues, active_workers);
+ rp->process_discovered_references(&is_alive, &keep_alive, &drain_queue, &par_task_executor);
+ }
+
+ // We have completed copying any necessary live referent objects
+ // (that were not copied during the actual pause) so we can
+ // retire any active alloc buffers
+ pss.retire_alloc_buffers();
+ assert(pss.refs()->is_empty(), "both queue and overflow should be empty");
+
+ // Any reference objects, in the collection set, that were 'discovered'
+ // by the CM ref processor should have already been copied (either by
+ // applying the external root copy closure to the discovered lists, or
+ // by following an RSet entry).
+ //
+ // But some of the referents, that are in the collection set, that these
+ // reference objects point to may not have been copied: the STW ref
+ // processor would have seen that the reference object had already
+ // been 'discovered' and would have skipped discovering the reference,
+ // but would not have treated the reference object as a regular oop.
+ // As a reult the copy closure would not have been applied to the
+ // referent object.
+ // We need to to copy these referent objects - the references will be
+ // processed at the end of Remark.
+
+ int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
+ workers()->total_workers() : 1);
+
+ set_par_threads(n_workers);
+ G1ParPreserveCMReferentsTask keep_cm_referents(this, n_workers, _task_queues);
+
+ if (G1CollectedHeap::use_parallel_gc_threads()) {
+ workers()->run_task(&keep_cm_referents);
+ } else {
+ keep_cm_referents.work(0);
+ }
+
+ set_par_threads(0);
+
+ double ref_proc_time = os::elapsedTime() - ref_proc_start;
+ g1_policy()->record_ref_proc_time(ref_proc_time * 1000.0);
+}
+
+// Weak Reference processing during an evacuation pause (part 2).
+void G1CollectedHeap::enqueue_discovered_references() {
+ double ref_enq_start = os::elapsedTime();
+
+ ReferenceProcessor* rp = _ref_processor_stw;
+ assert(!rp->discovery_enabled(), "should have been disabled as part of processing");
+
+ // Now enqueue any remaining on the discovered lists on to
+ // the pending list.
+ if (!rp->processing_is_mt()) {
+ // Serial reference processing...
+ rp->enqueue_discovered_references();
+ } else {
+ // Parallel reference enqueuing
+
+ int active_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1);
+ assert(rp->num_q() == active_workers, "sanity");
+ assert(active_workers <= rp->max_num_q(), "sanity");
+
+ G1STWRefProcTaskExecutor par_task_executor(this, workers(), _task_queues, active_workers);
+ rp->enqueue_discovered_references(&par_task_executor);
+ }
+
+ rp->verify_no_references_recorded();
+ assert(!rp->discovery_enabled(), "should have been disabled");
+
+ // FIXME
+ // CM's referent processing also cleans up the
+ // string table and symbol tables. Should we
+ // do that here also?
+
+ double ref_enq_time = os::elapsedTime() - ref_enq_start;
+ g1_policy()->record_ref_enq_time(ref_enq_time * 1000.0);
+}
+
void G1CollectedHeap::evacuate_collection_set() {
set_evacuation_failed(false);
g1_rem_set()->prepare_for_oops_into_collection_set_do();
concurrent_g1_refine()->set_use_cache(false);
@@ -4989,10 +5576,11 @@
rem_set()->prepare_for_younger_refs_iterate(true);
assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty");
double start_par = os::elapsedTime();
+
if (G1CollectedHeap::use_parallel_gc_threads()) {
// The individual threads will set their evac-failure closures.
StrongRootsScope srs(this);
if (ParallelGCVerbose) G1ParScanThreadState::print_termination_stats_hdr();
workers()->run_task(&g1_par_task);
@@ -5002,24 +5590,33 @@
}
double par_time = (os::elapsedTime() - start_par) * 1000.0;
g1_policy()->record_par_time(par_time);
set_par_threads(0);
+
+ // Process any discovered reference objects - we have
+ // to do this _before_ we retire the GC alloc regions
+ // as we may have to copy some 'reachable' referent
+ // objects (and their reachable sub-graphs) that were
+ // not copied during the pause.
+ process_discovered_references();
+
// Is this the right thing to do here? We don't save marks
// on individual heap regions when we allocate from
// them in parallel, so this seems like the correct place for this.
retire_all_alloc_regions();
// Weak root processing.
// Note: when JSR 292 is enabled and code blobs can contain
// non-perm oops then we will need to process the code blobs
// here too.
{
- G1IsAliveClosure is_alive(this);
+ G1STWIsAliveClosure is_alive(this);
G1KeepAliveClosure keep_alive(this);
JNIHandles::weak_oops_do(&is_alive, &keep_alive);
}
+
release_gc_alloc_regions(false /* totally */);
g1_rem_set()->cleanup_after_oops_into_collection_set_do();
concurrent_g1_refine()->clear_hot_cache();
concurrent_g1_refine()->set_use_cache(true);
@@ -5037,10 +5634,19 @@
} else if (PrintGC) {
gclog_or_tty->print("--");
}
}
+ // Enqueue any remaining references remaining on the STW
+ // reference processor's discovered lists. We need to do
+ // this after the card table is cleaned (and verified) as
+ // the act of enqueuing entries on to the pending list
+ // will log these updates (and dirty their associated
+ // cards). We need these updates logged to update any
+ // RSets.
+ enqueue_discovered_references();
+
if (G1DeferredRSUpdate) {
RedirtyLoggedCardTableEntryFastClosure redirty;
dirty_card_queue_set().set_closure(&redirty);
dirty_card_queue_set().apply_closure_to_all_completed_buffers();
@@ -5279,11 +5885,11 @@
// all regions and then clearing / dirtying as appropriate)
dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region());
}
double elapsed = os::elapsedTime() - start;
- g1_policy()->record_clear_ct_time( elapsed * 1000.0);
+ g1_policy()->record_clear_ct_time(elapsed * 1000.0);
#ifndef PRODUCT
if (G1VerifyCTCleanup || VerifyAfterGC) {
G1VerifyCardTableCleanup cleanup_verifier(this, ct_bs);
heap_region_iterate(&cleanup_verifier);
}