1 /*
2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shared/gcTraceTime.inline.hpp"
26 #include "gc/shared/workgroup.hpp"
27 #include "gc/shared/taskqueue.inline.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
30 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp"
31 #include "gc/shenandoah/shenandoahFreeSet.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
33 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
34 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
35 #include "gc/shenandoah/shenandoahPartialGC.hpp"
36 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
37 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
38 #include "memory/iterator.hpp"
39
40 class PartialEvacuateUpdateRootsClosure : public OopClosure {
41 ShenandoahPartialGC* _partial_gc;
42 Thread* _thread;
43 SCMObjToScanQueue* _queue;
44 private:
45 template <class T>
46 void do_oop_work(T* p) { _partial_gc->process_oop<T, false>(p, _thread, _queue); }
47 public:
48 PartialEvacuateUpdateRootsClosure(SCMObjToScanQueue* q) :
49 _partial_gc(ShenandoahHeap::heap()->partial_gc()),
50 _thread(Thread::current()), _queue(q) {}
51 void do_oop(oop* p) {
52 assert(! ShenandoahHeap::heap()->is_in_reserved(p), "sanity");
53 do_oop_work(p);
54 }
55 void do_oop(narrowOop* p) { do_oop_work(p); }
56 };
57
58 class ShenandoahPartialCollectionTask : public AbstractGangTask {
59 private:
60 ShenandoahRootProcessor* _rp;
61 ParallelTaskTerminator* _terminator;
62 ShenandoahHeapRegionSet* _root_regions;
63 ShenandoahHeap* _heap;
64 public:
65 ShenandoahPartialCollectionTask(ShenandoahRootProcessor* rp,
66 ParallelTaskTerminator* terminator,
67 ShenandoahHeapRegionSet* root_regions) :
68 AbstractGangTask("Shenandoah Partial Collection"),
69 _rp(rp), _terminator(terminator), _root_regions(root_regions),
70 _heap(ShenandoahHeap::heap()) {}
71
72 void work(uint worker_id) {
73 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
74 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues();
75 SCMObjToScanQueue* q = queues->queue(worker_id);
76
77 // Step 1: Process ordinary GC roots.
78 {
79 PartialEvacuateUpdateRootsClosure roots_cl(q);
80 CLDToOopClosure cld_cl(&roots_cl);
81 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
82 _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, worker_id);
83 }
84 if (check_and_handle_cancelled_gc()) return;
85
86 PartialEvacuateUpdateHeapClosure cl(q);
87
88 // Step 2: Process all root regions.
89 {
90 ShenandoahHeapRegion* r = _root_regions->claim_next();
91 while (r != NULL) {
92 assert(r->is_root(), "must be root region");
93 if (! r->is_humongous_continuation()) {
94 _heap->marked_object_oop_iterate(r, &cl);
95 }
96 r->set_root(false);
97 if (check_and_handle_cancelled_gc()) return;
98 r = _root_regions->claim_next();
99 }
100 }
101 if (check_and_handle_cancelled_gc()) return;
102
103 // Step 3: Drain all outstanding work in queues.
104 {
105 int seed = 17;
106 SCMTask task;
107
108 uint stride = ShenandoahMarkLoopStride;
109
110 while (true) {
111 if (check_and_handle_cancelled_gc()) return;
112
113 for (uint i = 0; i < stride; i++) {
114 if ((q->pop_buffer(task) ||
115 q->pop_local(task) ||
116 q->pop_overflow(task) ||
117 queues->steal(worker_id, &seed, task))) {
118 oop obj = task.obj();
119 assert(!oopDesc::is_null(obj), "must not be null");
120 obj->oop_iterate(&cl);
121 } else {
122 if (_terminator->offer_termination()) return;
123 }
124 }
125 }
126 }
127 }
128
129 private:
130 bool check_and_handle_cancelled_gc() {
131 if (_heap->cancelled_concgc()) {
132 ShenandoahCancelledTerminatorTerminator tt;
133 while (! _terminator->offer_termination(&tt));
134 return true;
135 }
136 return false;
137 }
138 };
139
140 ShenandoahPartialGC::ShenandoahPartialGC(ShenandoahHeap* heap, size_t max_regions) :
141 _heap(heap),
142 _matrix(heap->connection_matrix()),
143 _root_regions(new ShenandoahHeapRegionSet(max_regions)),
144 _task_queues(new SCMObjToScanQueueSet(heap->max_workers())) {
145
146 assert(_matrix != NULL, "need matrix");
147
148 uint num_queues = heap->max_workers();
149 for (uint i = 0; i < num_queues; ++i) {
150 SCMObjToScanQueue* task_queue = new SCMObjToScanQueue();
151 task_queue->initialize();
152 _task_queues->register_queue(i, task_queue);
153 }
154
155 }
156
157 void ShenandoahPartialGC::prepare() {
158
159 if (ShenandoahVerify || VerifyShenandoahMatrix) {
160 log_debug(gc, ergo)("Verifying before partial collection");
161 _heap->verify_heap_reachable_at_safepoint();
162 log_debug(gc, ergo)("Verification passed");
163 }
164
165 _heap->collection_set()->clear();
166 assert(_heap->collection_set()->count() == 0, "collection set not clear");
167
168 _heap->ensure_parsability(true);
169
170 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
171 ShenandoahHeapRegionSet* regions = _heap->regions();
172 ShenandoahCollectionSet* collection_set = _heap->collection_set();
173 ShenandoahFreeSet* free_set = _heap->free_regions();
174 free_set->clear();
175 _root_regions->clear();
176 assert(_root_regions->count() == 0, "must be cleared");
177 size_t num_regions = _heap->num_regions();
178
179 // First pass: find collection set.
180 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
181 ShenandoahHeapRegion* region = regions->get(to_idx);
182 region->set_root(false);
183 guarantee(! region->is_root(), "must not be root here");
184 if (region->is_humongous() || region->is_empty() || region->is_pinned()) continue;
185 assert(! _heap->region_in_collection_set(to_idx), "must not be in cset yet");
186 uint num_incoming = 0;
187 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
188 if (matrix->is_connected(from_idx, to_idx)) {
189 num_incoming++;
190 }
191 }
192 if (num_incoming < ShenandoahPartialInboundThreshold) {
193 collection_set->add_region(region);
194 _heap->set_region_in_collection_set(to_idx, true);
195 }
196 }
197
198 if (UseShenandoahMatrix && PrintShenandoahMatrix) {
199 outputStream* log = Log(gc)::info_stream();
200 matrix->print_on(log);
201 }
202
203 // Second pass: find all root regions.
204 size_t num_cset = collection_set->count();
205 for (uint idx = 0; idx < num_cset; idx++) {
206 ShenandoahHeapRegion* region = collection_set->get(idx);
207 size_t to_idx = region->region_number();
208 assert(! region->is_humongous() && ! region->is_empty() && ! region->is_pinned(), "sanity");
209 assert(_heap->region_in_collection_set(to_idx), "more sanity");
210 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
211 if (matrix->is_connected(from_idx, to_idx)) {
212 ShenandoahHeapRegion* r = regions->get(from_idx);
213 if (! r->is_root() && ! _heap->region_in_collection_set(from_idx)) {
214 _root_regions->add_region(r);
215 r->set_root(true);
216 }
217 }
218 }
219 }
220 // Final pass: free regions.
221 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
222 ShenandoahHeapRegion* region = regions->get(to_idx);
223 if (! region->is_humongous() &&
224 ! region->is_pinned() &&
225 ! region->is_root() &&
226 ! _heap->in_collection_set(region)) {
227
228 free_set->add_region(region);
229 }
230 if (region->is_root()) {
231 // We rebuild the outbound matrix for all roots, thus we can clear them before.
232 // This makes the matrix more precise.
233 matrix->clear_region_outbound(region->region_number());
234 }
235 }
236 log_debug(gc, ergo)("got "SIZE_FORMAT" cset regions", collection_set->count());
237 log_debug(gc, ergo)("got "SIZE_FORMAT" root regions", _root_regions->count());
238 }
239
240 void ShenandoahPartialGC::do_partial_collection() {
241
242 _heap->gc_timer()->register_gc_start();
243 ShenandoahCollectorPolicy* policy = _heap->shenandoahPolicy();
244 policy->record_phase_start(ShenandoahCollectorPolicy::partial_gc);
245
246 {
247 GCTraceTime(Info, gc) time("Pause Partial", _heap->gc_timer(), GCCause::_no_gc, true);
248
249 assert(_task_queues->is_empty(), "queues must be empty after partial GC");
250
251 COMPILER2_PRESENT(DerivedPointerTable::clear());
252
253 {
254 ClassLoaderDataGraph::clear_claimed_marks();
255 uint nworkers = _heap->workers()->active_workers();
256 ShenandoahRootProcessor rp(_heap, nworkers);
257
258 if (UseShenandoahOWST) {
259 ShenandoahTaskTerminator terminator(nworkers, task_queues());
260 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions);
261 _heap->workers()->run_task(&partial_task);
262 } else {
263 ParallelTaskTerminator terminator(nworkers, task_queues());
264 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions);
265 _heap->workers()->run_task(&partial_task);
266 }
267 }
268
269 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
270
271 if (_heap->cancelled_concgc()) {
272 _task_queues->clear();
273 }
274 assert(_task_queues->is_empty(), "queues must be empty after partial GC");
275
276 if (! _heap->cancelled_concgc()) {
277 if (ShenandoahVerify || VerifyShenandoahMatrix) {
278 log_debug(gc, ergo)("Verifying after partial collection");
279 _heap->verify_heap_reachable_at_safepoint();
280 log_debug(gc, ergo)("Verification passed");
281 }
282
283 ShenandoahHeap::ShenandoahHeapLock heap_lock(_heap);
284 size_t num_cset = _heap->collection_set()->count();
285 for (size_t i = 0; i < num_cset; i++) {
286 ShenandoahHeapRegion* r = _heap->collection_set()->get(i);
287 _heap->decrease_used(r->used());
288 HeapWord* bottom = r->bottom();
289 HeapWord* top = _heap->complete_top_at_mark_start(r->bottom());
290 if (top > bottom) {
291 _heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
292 }
293 r->recycle();
294 _heap->free_regions()->add_region(r);
295 }
296
297 reset();
298 }
299 }
300
301 policy->record_phase_end(ShenandoahCollectorPolicy::partial_gc);
302 _heap->gc_timer()->register_gc_end();
303 }
304
305 void ShenandoahPartialGC::reset() {
306 _heap->collection_set()->clear();
307 _heap->clear_cset_fast_test();
308 _root_regions->clear();
309 }
310
311 SCMObjToScanQueueSet* ShenandoahPartialGC::task_queues() {
312 return _task_queues;
313 }