< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp

Print this page
rev 50076 : Fold Partial GC into Traversal GC

*** 27,43 **** --- 27,45 ---- #include "gc/shared/markBitMap.inline.hpp" #include "gc/shared/workgroup.hpp" #include "gc/shared/taskqueue.inline.hpp" #include "gc/shared/weakProcessor.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" + #include "gc/shenandoah/shenandoahConnectionMatrix.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" + #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahTraversalGC.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" #include "gc/shenandoah/shenandoahStringDedup.hpp" #include "gc/shenandoah/shenandoahStrDedupQueue.inline.hpp"
*** 94,115 **** class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure { private: ShenandoahObjToScanQueue* _queue; ShenandoahTraversalGC* _traversal_gc; ShenandoahHeap* _heap; public: ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) : _queue(q), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), ! _heap(ShenandoahHeap::heap()) { } void do_buffer(void** buffer, size_t size) { for (size_t i = 0; i < size; ++i) { oop* p = (oop*) &buffer[i]; oop obj = RawAccess<>::oop_load(p); shenandoah_assert_not_forwarded(p, obj); ! if (!_heap->is_marked_next(obj) && _heap->mark_next(obj)) { _queue->push(ShenandoahMarkTask(obj)); } } } }; --- 96,120 ---- class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure { private: ShenandoahObjToScanQueue* _queue; ShenandoahTraversalGC* _traversal_gc; ShenandoahHeap* _heap; + ShenandoahHeapRegionSet* _traversal_set; + public: ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) : _queue(q), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), ! _heap(ShenandoahHeap::heap()), ! _traversal_set(ShenandoahHeap::heap()->traversal_gc()->traversal_set()) { } void do_buffer(void** buffer, size_t size) { for (size_t i = 0; i < size; ++i) { oop* p = (oop*) &buffer[i]; oop obj = RawAccess<>::oop_load(p); shenandoah_assert_not_forwarded(p, obj); ! if (_traversal_set->is_in((HeapWord*) obj) && !_heap->is_marked_next(obj) && _heap->mark_next(obj)) { _queue->push(ShenandoahMarkTask(obj)); } } } };
*** 297,307 **** } } ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) : _heap(heap), ! _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())) { uint num_queues = heap->max_workers(); for (uint i = 0; i < num_queues; ++i) { ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); task_queue->initialize(); --- 302,316 ---- } } ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) : _heap(heap), ! _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())), ! _traversal_set(new ShenandoahHeapRegionSet()), ! _root_regions(new ShenandoahHeapRegionSet()), ! _root_regions_iterator(_root_regions->iterator()), ! _matrix(heap->connection_matrix()) { uint num_queues = heap->max_workers(); for (uint i = 0; i < num_queues; ++i) { ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); task_queue->initialize();
*** 317,326 **** --- 326,364 ---- } ShenandoahTraversalGC::~ShenandoahTraversalGC() { } + void ShenandoahTraversalGC::prepare_regions() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t num_regions = heap->num_regions(); + ShenandoahConnectionMatrix* matrix = _heap->connection_matrix(); + + for (size_t i = 0; i < num_regions; i++) { + ShenandoahHeapRegion* region = heap->get_region(i); + if (heap->is_bitmap_slice_committed(region)) { + if (_traversal_set->is_in(i)) { + heap->set_next_top_at_mark_start(region->bottom(), region->top()); + region->clear_live_data(); + assert(heap->is_next_bitmap_clear_range(region->bottom(), region->end()), "bitmap for traversal regions must be cleared"); + } else { + // Everything outside the traversal set is always considered live. + heap->set_next_top_at_mark_start(region->bottom(), region->bottom()); + } + if (_root_regions->is_in(i)) { + assert(!region->in_collection_set(), "roots must not overlap with cset"); + matrix->clear_region_outbound(i); + // Since root region can be allocated at, we should bound the scans + // in it at current top. Otherwise, one thread may evacuate objects + // to that root region, while another would try to scan newly evac'ed + // objects under the race. + region->set_concurrent_iteration_safe_limit(region->top()); + } + } + } + } + void ShenandoahTraversalGC::prepare() { _heap->collection_set()->clear(); assert(_heap->collection_set()->count() == 0, "collection set not clear"); _heap->make_tlabs_parsable(true);
*** 329,344 **** ShenandoahFreeSet* free_set = _heap->free_set(); ShenandoahCollectionSet* collection_set = _heap->collection_set(); // Find collection set ! _heap->shenandoahPolicy()->choose_collection_set(collection_set, false); // Rebuild free set free_set->rebuild(); ! log_info(gc,ergo)("Got "SIZE_FORMAT" collection set regions", collection_set->count()); } void ShenandoahTraversalGC::init_traversal_collection() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC"); --- 367,383 ---- ShenandoahFreeSet* free_set = _heap->free_set(); ShenandoahCollectionSet* collection_set = _heap->collection_set(); // Find collection set ! _heap->shenandoahPolicy()->choose_collection_set(collection_set); ! prepare_regions(); // Rebuild free set free_set->rebuild(); ! log_info(gc,ergo)("Got "SIZE_FORMAT" collection set regions and "SIZE_FORMAT" root set regions", collection_set->count(), _root_regions->count()); } void ShenandoahTraversalGC::init_traversal_collection() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC");
*** 391,400 **** --- 430,441 ---- } if (ShenandoahPacing) { _heap->pacer()->setup_for_traversal(); } + + _root_regions_iterator = _root_regions->iterator(); } void ShenandoahTraversalGC::main_loop(uint worker_id, ParallelTaskTerminator* terminator, bool do_satb) { if (do_satb) { main_loop_prework<true>(worker_id, terminator);
*** 413,422 **** --- 454,506 ---- ReferenceProcessor* rp = NULL; if (_heap->process_references()) { rp = _heap->ref_processor(); } + if (UseShenandoahMatrix) { + if (!_heap->is_degenerated_gc_in_progress()) { + if (_heap->unload_classes()) { + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); + ShenandoahTraversalMetadataDedupMatrixClosure cl(q, rp, dq); + main_loop_work<ShenandoahTraversalMetadataDedupMatrixClosure, DO_SATB>(&cl, ld, w, t); + } else { + ShenandoahTraversalMetadataMatrixClosure cl(q, rp); + main_loop_work<ShenandoahTraversalMetadataMatrixClosure, DO_SATB>(&cl, ld, w, t); + } + } else { + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); + ShenandoahTraversalDedupMatrixClosure cl(q, rp, dq); + main_loop_work<ShenandoahTraversalDedupMatrixClosure, DO_SATB>(&cl, ld, w, t); + } else { + ShenandoahTraversalMatrixClosure cl(q, rp); + main_loop_work<ShenandoahTraversalMatrixClosure, DO_SATB>(&cl, ld, w, t); + } + } + } else { + if (_heap->unload_classes()) { + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); + ShenandoahTraversalMetadataDedupDegenMatrixClosure cl(q, rp, dq); + main_loop_work<ShenandoahTraversalMetadataDedupDegenMatrixClosure, DO_SATB>(&cl, ld, w, t); + } else { + ShenandoahTraversalMetadataDegenMatrixClosure cl(q, rp); + main_loop_work<ShenandoahTraversalMetadataDegenMatrixClosure, DO_SATB>(&cl, ld, w, t); + } + } else { + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); + ShenandoahTraversalDedupDegenMatrixClosure cl(q, rp, dq); + main_loop_work<ShenandoahTraversalDedupDegenMatrixClosure, DO_SATB>(&cl, ld, w, t); + } else { + ShenandoahTraversalDegenMatrixClosure cl(q, rp); + main_loop_work<ShenandoahTraversalDegenMatrixClosure, DO_SATB>(&cl, ld, w, t); + } + } + } + } else { if (!_heap->is_degenerated_gc_in_progress()) { if (_heap->unload_classes()) { if (ShenandoahStringDedup::is_enabled()) { ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); ShenandoahTraversalMetadataDedupClosure cl(q, rp, dq);
*** 454,463 **** --- 538,548 ---- ShenandoahTraversalDegenClosure cl(q, rp); main_loop_work<ShenandoahTraversalDegenClosure, DO_SATB>(&cl, ld, w, t); } } } + } flush_liveness(w); } template <class T, bool DO_SATB>
*** 490,499 **** --- 575,601 ---- q = queues->claim_next(); break; } } } + + if (check_and_handle_cancelled_gc(terminator)) return; + + // Step 2: Process all root regions. + // TODO: Interleave this in the normal mark loop below. + ShenandoahHeapRegion* r = _root_regions_iterator.claim_next(); + while (r != NULL) { + _heap->marked_object_oop_safe_iterate(r, cl); + if (ShenandoahPacing) { + _heap->pacer()->report_partial(r->get_live_data_words()); + } + if (check_and_handle_cancelled_gc(terminator)) return; + r = _root_regions_iterator.claim_next(); + } + + if (check_and_handle_cancelled_gc(terminator)) return; + // Normal loop. q = queues->queue(worker_id); ShenandoahTraversalSATBBufferClosure satb_cl(q); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
*** 501,515 **** while (true) { if (check_and_handle_cancelled_gc(terminator)) return; for (uint i = 0; i < stride; i++) { ! if ((q->pop_buffer(task) || q->pop_local(task) || q->pop_overflow(task) || (DO_SATB && satb_mq_set.apply_closure_to_completed_buffer(&satb_cl) && q->pop_buffer(task)) || ! queues->steal(worker_id, &seed, task))) { conc_mark->do_task<T, true>(q, cl, live_data, &task); } else { ShenandoahEvacOOMScopeLeaver oom_scope_leaver; if (terminator->offer_termination()) return; } --- 603,617 ---- while (true) { if (check_and_handle_cancelled_gc(terminator)) return; for (uint i = 0; i < stride; i++) { ! if (q->pop_buffer(task) || q->pop_local(task) || q->pop_overflow(task) || (DO_SATB && satb_mq_set.apply_closure_to_completed_buffer(&satb_cl) && q->pop_buffer(task)) || ! queues->steal(worker_id, &seed, task)) { conc_mark->do_task<T, true>(q, cl, live_data, &task); } else { ShenandoahEvacOOMScopeLeaver oom_scope_leaver; if (terminator->offer_termination()) return; }
*** 596,622 **** // Trash everything // Clear immediate garbage regions. size_t num_regions = _heap->num_regions(); ShenandoahFreeSet* free_regions = _heap->free_set(); free_regions->clear(); for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* r = _heap->get_region(i); bool not_allocated = _heap->next_top_at_mark_start(r->bottom()) == r->top(); ! if (r->is_humongous_start() && !r->has_live() && not_allocated) { // Trash humongous. HeapWord* humongous_obj = r->bottom() + BrooksPointer::word_size(); assert(!_heap->is_marked_next(oop(humongous_obj)), "must not be marked"); r->make_trash(); while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) { i++; r = _heap->get_region(i); assert(r->is_humongous_continuation(), "must be humongous continuation"); r->make_trash(); } ! } else if (!r->is_empty() && !r->has_live() && not_allocated) { // Trash regular. assert(!r->is_humongous(), "handled above"); assert(!r->is_trash(), "must not already be trashed"); r->make_trash(); } --- 698,727 ---- // Trash everything // Clear immediate garbage regions. size_t num_regions = _heap->num_regions(); + ShenandoahHeapRegionSet* traversal_regions = traversal_set(); ShenandoahFreeSet* free_regions = _heap->free_set(); free_regions->clear(); for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* r = _heap->get_region(i); bool not_allocated = _heap->next_top_at_mark_start(r->bottom()) == r->top(); ! ! bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated; ! if (r->is_humongous_start() && candidate) { // Trash humongous. HeapWord* humongous_obj = r->bottom() + BrooksPointer::word_size(); assert(!_heap->is_marked_next(oop(humongous_obj)), "must not be marked"); r->make_trash(); while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) { i++; r = _heap->get_region(i); assert(r->is_humongous_continuation(), "must be humongous continuation"); r->make_trash(); } ! } else if (!r->is_empty() && candidate) { // Trash regular. assert(!r->is_humongous(), "handled above"); assert(!r->is_trash(), "must not already be trashed"); r->make_trash(); }
*** 724,734 **** ShenandoahObjToScanQueue* _queue; Thread* _thread; ShenandoahTraversalGC* _traversal_gc; template <class T> inline void do_oop_nv(T* p) { ! _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue); } public: ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : _queue(q), _thread(Thread::current()), --- 829,839 ---- ShenandoahObjToScanQueue* _queue; Thread* _thread; ShenandoahTraversalGC* _traversal_gc; template <class T> inline void do_oop_nv(T* p) { ! _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, false /* matrix */>(p, _thread, _queue, NULL); } public: ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : _queue(q), _thread(Thread::current()),
*** 743,753 **** ShenandoahObjToScanQueue* _queue; Thread* _thread; ShenandoahTraversalGC* _traversal_gc; template <class T> inline void do_oop_nv(T* p) { ! _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue); } public: ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) : _queue(q), _thread(Thread::current()), --- 848,858 ---- ShenandoahObjToScanQueue* _queue; Thread* _thread; ShenandoahTraversalGC* _traversal_gc; template <class T> inline void do_oop_nv(T* p) { ! _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* matrix */>(p, _thread, _queue, NULL); } public: ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) : _queue(q), _thread(Thread::current()),
*** 755,764 **** --- 860,909 ---- void do_oop(narrowOop* p) { do_oop_nv(p); } void do_oop(oop* p) { do_oop_nv(p); } }; + class ShenandoahTraversalKeepAliveUpdateMatrixClosure : public OopClosure { + private: + ShenandoahObjToScanQueue* _queue; + Thread* _thread; + ShenandoahTraversalGC* _traversal_gc; + template <class T> + inline void do_oop_nv(T* p) { + // TODO: Need to somehow pass base_obj here? + _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* matrix */>(p, _thread, _queue, NULL); + } + + public: + ShenandoahTraversalKeepAliveUpdateMatrixClosure(ShenandoahObjToScanQueue* q) : + _queue(q), _thread(Thread::current()), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {} + + void do_oop(narrowOop* p) { do_oop_nv(p); } + void do_oop(oop* p) { do_oop_nv(p); } + }; + + class ShenandoahTraversalKeepAliveUpdateDegenMatrixClosure : public OopClosure { + private: + ShenandoahObjToScanQueue* _queue; + Thread* _thread; + ShenandoahTraversalGC* _traversal_gc; + template <class T> + inline void do_oop_nv(T* p) { + // TODO: Need to somehow pass base_obj here? + _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, true /* matrix */>(p, _thread, _queue, NULL); + } + + public: + ShenandoahTraversalKeepAliveUpdateDegenMatrixClosure(ShenandoahObjToScanQueue* q) : + _queue(q), _thread(Thread::current()), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {} + + void do_oop(narrowOop* p) { do_oop_nv(p); } + void do_oop(oop* p) { do_oop_nv(p); } + }; + void ShenandoahTraversalGC::preclean_weak_refs() { // Pre-cleaning weak references before diving into STW makes sense at the // end of concurrent mark. This will filter out the references which referents // are alive. Note that ReferenceProcessor already filters out these on reference // discovery, and the bulk of work is done here. This phase processes leftovers
*** 786,800 **** --- 931,953 ---- assert(task_queues()->is_empty(), "Should be empty"); assert(!sh->is_degenerated_gc_in_progress(), "must be in concurrent non-degenerated phase"); ShenandoahTraversalPrecleanCompleteGCClosure complete_gc; ShenandoahForwardedIsAliveClosure is_alive; + if (UseShenandoahMatrix) { + ShenandoahTraversalKeepAliveUpdateMatrixClosure keep_alive(task_queues()->queue(0)); + ResourceMark rm; + rp->preclean_discovered_references(&is_alive, &keep_alive, + &complete_gc, &yield, + NULL); + } else { ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(0)); ResourceMark rm; rp->preclean_discovered_references(&is_alive, &keep_alive, &complete_gc, &yield, NULL); + } assert(!sh->cancelled_concgc() || task_queues()->is_empty(), "Should be empty"); } // Weak Reference Closures class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
*** 865,882 **** --- 1018,1045 ---- assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator); ShenandoahForwardedIsAliveClosure is_alive; + if (UseShenandoahMatrix) { + if (!heap->is_degenerated_gc_in_progress()) { + ShenandoahTraversalKeepAliveUpdateMatrixClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id)); + _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); + } else { + ShenandoahTraversalKeepAliveUpdateDegenMatrixClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id)); + _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); + } + } else { if (!heap->is_degenerated_gc_in_progress()) { ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id)); _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); } else { ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id)); _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); } } + } }; class ShenandoahTraversalRefEnqueueTaskProxy : public AbstractGangTask { private:
*** 973,989 **** { ShenandoahGCPhase phase(phase_process); ShenandoahForwardedIsAliveClosure is_alive; ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id)); rp->process_discovered_references(&is_alive, &keep_alive, &complete_gc, &executor, &pt); pt.print_all_references(); - WeakProcessor::weak_oops_do(&is_alive, &keep_alive); assert(!_heap->cancelled_concgc() || task_queues()->is_empty(), "Should be empty"); } if (_heap->cancelled_concgc()) return; --- 1136,1178 ---- { ShenandoahGCPhase phase(phase_process); ShenandoahForwardedIsAliveClosure is_alive; + if (UseShenandoahMatrix) { + if (!_heap->is_degenerated_gc_in_progress()) { + ShenandoahTraversalKeepAliveUpdateMatrixClosure keep_alive(task_queues()->queue(serial_worker_id)); + rp->process_discovered_references(&is_alive, &keep_alive, + &complete_gc, &executor, + &pt); + pt.print_all_references(); + WeakProcessor::weak_oops_do(&is_alive, &keep_alive); + } else { + ShenandoahTraversalKeepAliveUpdateDegenMatrixClosure keep_alive(task_queues()->queue(serial_worker_id)); + rp->process_discovered_references(&is_alive, &keep_alive, + &complete_gc, &executor, + &pt); + pt.print_all_references(); + WeakProcessor::weak_oops_do(&is_alive, &keep_alive); + } + } else { + if (!_heap->is_degenerated_gc_in_progress()) { ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id)); rp->process_discovered_references(&is_alive, &keep_alive, &complete_gc, &executor, &pt); pt.print_all_references(); WeakProcessor::weak_oops_do(&is_alive, &keep_alive); + } else { + ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(task_queues()->queue(serial_worker_id)); + rp->process_discovered_references(&is_alive, &keep_alive, + &complete_gc, &executor, + &pt); + pt.print_all_references(); + WeakProcessor::weak_oops_do(&is_alive, &keep_alive); + } + } assert(!_heap->cancelled_concgc() || task_queues()->is_empty(), "Should be empty"); } if (_heap->cancelled_concgc()) return;
< prev index next >