22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shared/referenceProcessor.hpp"
27 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
28 #include "gc/shared/workgroup.hpp"
29 #include "gc/shared/weakProcessor.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
32 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
33 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
34 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
35 #include "gc/shenandoah/shenandoahFreeSet.hpp"
36 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
37 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
38 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
39 #include "gc/shenandoah/shenandoahHeuristics.hpp"
40 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
41 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
42 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
43 #include "gc/shenandoah/shenandoahStringDedup.hpp"
44 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
45 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
46 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
47 #include "gc/shenandoah/shenandoahUtils.hpp"
48 #include "gc/shenandoah/shenandoahVerifier.hpp"
49
50 #include "memory/iterator.hpp"
51 #include "memory/metaspace.hpp"
52 #include "memory/resourceArea.hpp"
53
54 /**
55 * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
56 * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
57 * is incremental-update-based.
58 *
59 * NOTE on interaction with TAMS: we want to avoid traversing new objects for
60 * several reasons:
61 * - We will not reclaim them in this cycle anyway, because they are not in the
62 * cset
175 void work(uint worker_id) {
176 ShenandoahParallelWorkerSession worker_session(worker_id);
177
178 ShenandoahEvacOOMScope oom_evac_scope;
179 ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
180 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
181
182 bool process_refs = _heap->process_references();
183 bool unload_classes = _heap->unload_classes();
184 ReferenceProcessor* rp = NULL;
185 if (process_refs) {
186 rp = _heap->ref_processor();
187 }
188
189 // Step 1: Process ordinary GC roots.
190 {
191 ShenandoahTraversalClosure roots_cl(q, rp);
192 ShenandoahMarkCLDClosure cld_cl(&roots_cl);
193 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
194 if (unload_classes) {
195 _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, NULL, NULL, worker_id);
196 // Need to pre-evac code roots here. Otherwise we might see from-space constants.
197 ShenandoahWorkerTimings* worker_times = _heap->phase_timings()->worker_times();
198 ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
199 _cset_coderoots->possibly_parallel_blobs_do(&code_cl);
200 } else {
201 _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, NULL, worker_id);
202 }
203 }
204 }
205 };
206
207 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
208 private:
209 ShenandoahTaskTerminator* _terminator;
210 ShenandoahHeap* _heap;
211 public:
212 ShenandoahConcurrentTraversalCollectionTask(ShenandoahTaskTerminator* terminator) :
213 AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
214 _terminator(terminator),
215 _heap(ShenandoahHeap::heap()) {}
216
217 void work(uint worker_id) {
218 ShenandoahConcurrentWorkerSession worker_session(worker_id);
219 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
220 ShenandoahEvacOOMScope oom_evac_scope;
221 ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
256 // Step 0: Drain outstanding SATB queues.
257 // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
258 ShenandoahTraversalSATBBufferClosure satb_cl(q);
259 {
260 // Process remaining finished SATB buffers.
261 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
262 while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
263 // Process remaining threads SATB buffers below.
264 }
265
266 // Step 1: Process GC roots.
267 // For oops in code roots, they are marked, evacuated, enqueued for further traversal,
268 // and the references to the oops are updated during init pause. New nmethods are handled
269 // in similar way during nmethod-register process. Therefore, we don't need to rescan code
270 // roots here.
271 if (!_heap->is_degenerated_gc_in_progress()) {
272 ShenandoahTraversalClosure roots_cl(q, rp);
273 CLDToOopClosure cld_cl(&roots_cl);
274 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
275 if (unload_classes) {
276 ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
277 _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, NULL, &tc, worker_id);
278 } else {
279 _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, &tc, worker_id);
280 }
281 } else {
282 ShenandoahTraversalDegenClosure roots_cl(q, rp);
283 CLDToOopClosure cld_cl(&roots_cl);
284 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
285 if (unload_classes) {
286 ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
287 _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, NULL, &tc, worker_id);
288 } else {
289 _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, &tc, worker_id);
290 }
291 }
292
293 {
294 ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
295 ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
296
297 // Step 3: Finally drain all outstanding work in queues.
298 traversal_gc->main_loop(worker_id, _terminator, false);
299 }
300
301 }
302 };
303
304 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
305 _heap(heap),
306 _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),
307 _traversal_set(ShenandoahHeapRegionSet()) {
308
309 uint num_queues = heap->max_workers();
588 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
589 uint nworkers = _heap->workers()->active_workers();
590 task_queues()->reserve(nworkers);
591
592 // Finish traversal
593 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
594 ShenandoahTerminationTracker term(ShenandoahPhaseTimings::final_traversal_gc_termination);
595
596 ShenandoahTaskTerminator terminator(nworkers, task_queues());
597 ShenandoahFinalTraversalCollectionTask task(&rp, &terminator);
598 _heap->workers()->run_task(&task);
599 #if defined(COMPILER2) || INCLUDE_JVMCI
600 DerivedPointerTable::update_pointers();
601 #endif
602 }
603
604 if (!_heap->cancelled_gc() && _heap->process_references()) {
605 weak_refs_work();
606 }
607
608 if (!_heap->cancelled_gc() && _heap->unload_classes()) {
609 _heap->unload_classes_and_cleanup_tables(false);
610 fixup_roots();
611 }
612
613 if (!_heap->cancelled_gc()) {
614 assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
615 TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats());
616 TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
617
618 // No more marking expected
619 _heap->mark_complete_marking_context();
620
621 // Resize metaspace
622 MetaspaceGC::compute_new_size();
623
624 // Still good? We can now trash the cset, and make final verification
625 {
626 ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
627 ShenandoahHeapLocker lock(_heap->lock());
628
629 // Trash everything
685 oop obj = CompressedOops::decode_not_null(o);
686 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
687 if (!oopDesc::unsafe_equals(obj, forw)) {
688 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
689 }
690 }
691 }
692
693 public:
694 inline void do_oop(oop* p) { do_oop_work(p); }
695 inline void do_oop(narrowOop* p) { do_oop_work(p); }
696 };
697
698 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
699 private:
700 ShenandoahRootProcessor* _rp;
701
702 public:
703 ShenandoahTraversalFixRootsTask(ShenandoahRootProcessor* rp) :
704 AbstractGangTask("Shenandoah traversal fix roots"),
705 _rp(rp) {}
706
707 void work(uint worker_id) {
708 ShenandoahParallelWorkerSession worker_session(worker_id);
709 ShenandoahTraversalFixRootsClosure cl;
710 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
711 CLDToOopClosure cldCl(&cl);
712 _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id);
713 }
714 };
715
716 void ShenandoahTraversalGC::fixup_roots() {
717 #if defined(COMPILER2) || INCLUDE_JVMCI
718 DerivedPointerTable::clear();
719 #endif
720 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots);
721 ShenandoahTraversalFixRootsTask update_roots_task(&rp);
722 _heap->workers()->run_task(&update_roots_task);
723 #if defined(COMPILER2) || INCLUDE_JVMCI
724 DerivedPointerTable::update_pointers();
725 #endif
726 }
727
728 void ShenandoahTraversalGC::reset() {
729 _task_queues->clear();
730 }
731
732 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
|
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shared/referenceProcessor.hpp"
27 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
28 #include "gc/shared/workgroup.hpp"
29 #include "gc/shared/weakProcessor.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
32 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
33 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
34 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
35 #include "gc/shenandoah/shenandoahFreeSet.hpp"
36 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
37 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
38 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
39 #include "gc/shenandoah/shenandoahHeuristics.hpp"
40 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
41 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
42 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
43 #include "gc/shenandoah/shenandoahStringDedup.hpp"
44 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
45 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
46 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
47 #include "gc/shenandoah/shenandoahUtils.hpp"
48 #include "gc/shenandoah/shenandoahVerifier.hpp"
49
50 #include "memory/iterator.hpp"
51 #include "memory/metaspace.hpp"
52 #include "memory/resourceArea.hpp"
53
54 /**
55 * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
56 * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
57 * is incremental-update-based.
58 *
59 * NOTE on interaction with TAMS: we want to avoid traversing new objects for
60 * several reasons:
61 * - We will not reclaim them in this cycle anyway, because they are not in the
62 * cset
175 void work(uint worker_id) {
176 ShenandoahParallelWorkerSession worker_session(worker_id);
177
178 ShenandoahEvacOOMScope oom_evac_scope;
179 ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
180 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
181
182 bool process_refs = _heap->process_references();
183 bool unload_classes = _heap->unload_classes();
184 ReferenceProcessor* rp = NULL;
185 if (process_refs) {
186 rp = _heap->ref_processor();
187 }
188
189 // Step 1: Process ordinary GC roots.
190 {
191 ShenandoahTraversalClosure roots_cl(q, rp);
192 ShenandoahMarkCLDClosure cld_cl(&roots_cl);
193 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
194 if (unload_classes) {
195 _rp->process_strong_roots(&roots_cl, &cld_cl, NULL, NULL, worker_id);
196 // Need to pre-evac code roots here. Otherwise we might see from-space constants.
197 ShenandoahWorkerTimings* worker_times = _heap->phase_timings()->worker_times();
198 ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
199 _cset_coderoots->possibly_parallel_blobs_do(&code_cl);
200 } else {
201 _rp->process_all_roots(&roots_cl, &cld_cl, &code_cl, NULL, worker_id);
202 }
203 }
204 }
205 };
206
207 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
208 private:
209 ShenandoahTaskTerminator* _terminator;
210 ShenandoahHeap* _heap;
211 public:
212 ShenandoahConcurrentTraversalCollectionTask(ShenandoahTaskTerminator* terminator) :
213 AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
214 _terminator(terminator),
215 _heap(ShenandoahHeap::heap()) {}
216
217 void work(uint worker_id) {
218 ShenandoahConcurrentWorkerSession worker_session(worker_id);
219 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
220 ShenandoahEvacOOMScope oom_evac_scope;
221 ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
256 // Step 0: Drain outstanding SATB queues.
257 // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
258 ShenandoahTraversalSATBBufferClosure satb_cl(q);
259 {
260 // Process remaining finished SATB buffers.
261 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
262 while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
263 // Process remaining threads SATB buffers below.
264 }
265
266 // Step 1: Process GC roots.
267 // For oops in code roots, they are marked, evacuated, enqueued for further traversal,
268 // and the references to the oops are updated during init pause. New nmethods are handled
269 // in similar way during nmethod-register process. Therefore, we don't need to rescan code
270 // roots here.
271 if (!_heap->is_degenerated_gc_in_progress()) {
272 ShenandoahTraversalClosure roots_cl(q, rp);
273 CLDToOopClosure cld_cl(&roots_cl);
274 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
275 if (unload_classes) {
276 ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
277 _rp->process_strong_roots(&roots_cl, &remark_cld_cl, NULL, &tc, worker_id);
278 } else {
279 _rp->process_all_roots(&roots_cl, &cld_cl, NULL, &tc, worker_id);
280 }
281 } else {
282 ShenandoahTraversalDegenClosure roots_cl(q, rp);
283 CLDToOopClosure cld_cl(&roots_cl);
284 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
285 if (unload_classes) {
286 ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
287 _rp->process_strong_roots(&roots_cl, &remark_cld_cl, NULL, &tc, worker_id);
288 } else {
289 _rp->process_all_roots(&roots_cl, &cld_cl, NULL, &tc, worker_id);
290 }
291 }
292
293 {
294 ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
295 ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
296
297 // Step 3: Finally drain all outstanding work in queues.
298 traversal_gc->main_loop(worker_id, _terminator, false);
299 }
300
301 }
302 };
303
304 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
305 _heap(heap),
306 _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),
307 _traversal_set(ShenandoahHeapRegionSet()) {
308
309 uint num_queues = heap->max_workers();
588 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
589 uint nworkers = _heap->workers()->active_workers();
590 task_queues()->reserve(nworkers);
591
592 // Finish traversal
593 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
594 ShenandoahTerminationTracker term(ShenandoahPhaseTimings::final_traversal_gc_termination);
595
596 ShenandoahTaskTerminator terminator(nworkers, task_queues());
597 ShenandoahFinalTraversalCollectionTask task(&rp, &terminator);
598 _heap->workers()->run_task(&task);
599 #if defined(COMPILER2) || INCLUDE_JVMCI
600 DerivedPointerTable::update_pointers();
601 #endif
602 }
603
604 if (!_heap->cancelled_gc() && _heap->process_references()) {
605 weak_refs_work();
606 }
607
608 if (!_heap->cancelled_gc()) {
609 if (_heap->unload_classes()) {
610 _heap->unload_classes_and_cleanup_tables(false);
611 }
612
613 fixup_roots();
614 }
615
616 if (!_heap->cancelled_gc()) {
617 assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
618 TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats());
619 TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
620
621 // No more marking expected
622 _heap->mark_complete_marking_context();
623
624 // Resize metaspace
625 MetaspaceGC::compute_new_size();
626
627 // Still good? We can now trash the cset, and make final verification
628 {
629 ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
630 ShenandoahHeapLocker lock(_heap->lock());
631
632 // Trash everything
688 oop obj = CompressedOops::decode_not_null(o);
689 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
690 if (!oopDesc::unsafe_equals(obj, forw)) {
691 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
692 }
693 }
694 }
695
696 public:
697 inline void do_oop(oop* p) { do_oop_work(p); }
698 inline void do_oop(narrowOop* p) { do_oop_work(p); }
699 };
700
701 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
702 private:
703 ShenandoahRootProcessor* _rp;
704
705 public:
706 ShenandoahTraversalFixRootsTask(ShenandoahRootProcessor* rp) :
707 AbstractGangTask("Shenandoah traversal fix roots"),
708 _rp(rp) {
709 assert(ShenandoahHeap::heap()->has_forwarded_objects(), "Must be");
710 }
711
712 void work(uint worker_id) {
713 ShenandoahParallelWorkerSession worker_session(worker_id);
714 ShenandoahTraversalFixRootsClosure cl;
715 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
716 CLDToOopClosure cldCl(&cl);
717 _rp->update_all_roots<ShenandoahForwardedIsAliveClosure>(&cl, &cldCl, &blobsCl, NULL, worker_id);
718 }
719 };
720
721 void ShenandoahTraversalGC::fixup_roots() {
722 #if defined(COMPILER2) || INCLUDE_JVMCI
723 DerivedPointerTable::clear();
724 #endif
725 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots);
726 ShenandoahTraversalFixRootsTask update_roots_task(&rp);
727 _heap->workers()->run_task(&update_roots_task);
728 #if defined(COMPILER2) || INCLUDE_JVMCI
729 DerivedPointerTable::update_pointers();
730 #endif
731 }
732
733 void ShenandoahTraversalGC::reset() {
734 _task_queues->clear();
735 }
736
737 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
|