1 /*
2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shared/gcTraceTime.inline.hpp"
27 #include "gc/shared/workgroup.hpp"
28 #include "gc/shared/taskqueue.inline.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
31 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
32 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp"
33 #include "gc/shenandoah/shenandoahFreeSet.hpp"
34 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
35 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
36 #include "gc/shenandoah/shenandoahHeap.hpp"
37 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
38 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
39 #include "gc/shenandoah/shenandoahPartialGC.hpp"
40 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
41 #include "gc/shenandoah/shenandoahStringDedup.hpp"
42 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
43 #include "gc/shenandoah/shenandoahUtils.hpp"
44 #include "gc/shenandoah/shenandoahVerifier.hpp"
45 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
46 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
47
48 #include "memory/iterator.hpp"
49 #include "runtime/safepoint.hpp"
50
51 class ShenandoahPartialEvacuateUpdateRootsClosure : public OopClosure {
52 ShenandoahPartialGC* _partial_gc;
53 Thread* _thread;
54 ShenandoahObjToScanQueue* _queue;
55 private:
56 template <class T>
57 void do_oop_work(T* p) { _partial_gc->process_oop<T, false>(p, _thread, _queue); }
58 public:
59 ShenandoahPartialEvacuateUpdateRootsClosure(ShenandoahObjToScanQueue* q) :
60 _partial_gc(ShenandoahHeap::heap()->partial_gc()),
61 _thread(Thread::current()), _queue(q) {}
62 void do_oop(oop* p) {
63 assert(! ShenandoahHeap::heap()->is_in_reserved(p), "sanity");
64 do_oop_work(p);
65 }
66 void do_oop(narrowOop* p) { do_oop_work(p); }
67 };
68
69 class ShenandoahPartialCollectionTask : public AbstractGangTask {
70 private:
71 ShenandoahRootProcessor* _rp;
72 ParallelTaskTerminator* _terminator;
73 ShenandoahHeapRegionSet* _root_regions;
74 ShenandoahHeap* _heap;
75 public:
76 ShenandoahPartialCollectionTask(ShenandoahRootProcessor* rp,
77 ParallelTaskTerminator* terminator,
78 ShenandoahHeapRegionSet* root_regions) :
79 AbstractGangTask("Shenandoah Partial Collection"),
80 _rp(rp), _terminator(terminator), _root_regions(root_regions),
81 _heap(ShenandoahHeap::heap()) {}
82
83 void work(uint worker_id) {
84 ShenandoahObjToScanQueueSet* queues = _heap->partial_gc()->task_queues();
85 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
86
87 // Step 1: Process ordinary GC roots.
88 {
89 ShenandoahPartialEvacuateUpdateRootsClosure roots_cl(q);
90 CLDToOopClosure cld_cl(&roots_cl);
91 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
92 _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, worker_id);
93 }
94 if (check_and_handle_cancelled_gc()) return;
95
96 ShenandoahPartialEvacuateUpdateHeapClosure cl(q);
97
98 // Step 2: Process all root regions.
99 {
100 ShenandoahHeapRegion* r = _root_regions->claim_next();
101 while (r != NULL) {
102 assert(r->is_root(), "must be root region");
103 _heap->marked_object_oop_safe_iterate(r, &cl);
104 if (check_and_handle_cancelled_gc()) return;
105 r = _root_regions->claim_next();
106 }
107 }
108 if (check_and_handle_cancelled_gc()) return;
109
110 // Step 3: Drain all outstanding work in queues.
111 {
112 int seed = 17;
113 ShenandoahMarkTask task;
114
115 uintx stride = ShenandoahMarkLoopStride;
116
117 while (true) {
118 if (check_and_handle_cancelled_gc()) return;
119
120 for (uint i = 0; i < stride; i++) {
121 if ((q->pop_buffer(task) ||
122 q->pop_local(task) ||
123 q->pop_overflow(task) ||
124 queues->steal(worker_id, &seed, task))) {
125 oop obj = task.obj();
126 assert(!oopDesc::is_null(obj), "must not be null");
127 obj->oop_iterate(&cl);
128 } else {
129 if (_terminator->offer_termination()) return;
130 }
131 }
132 }
133 }
134 }
135
136 private:
137 bool check_and_handle_cancelled_gc() {
138 if (_heap->cancelled_concgc()) {
139 ShenandoahCancelledTerminatorTerminator tt;
140 while (! _terminator->offer_termination(&tt));
141 return true;
142 }
143 return false;
144 }
145 };
146
147 class ShenandoahPartialCollectionCleanupTask : public AbstractGangTask {
148 private:
149 ShenandoahHeap* _heap;
150 public:
151 ShenandoahPartialCollectionCleanupTask() :
152 AbstractGangTask("Shenandoah Partial Collection Cleanup"),
153 _heap(ShenandoahHeap::heap()) {
154 _heap->collection_set()->clear_current_index();
155 }
156
157 void work(uint worker_id) {
158 ShenandoahCollectionSet* cset = _heap->collection_set();
159 ShenandoahHeapRegion* r = cset->claim_next();
160 while (r != NULL) {
161 HeapWord* bottom = r->bottom();
162 HeapWord* top = _heap->complete_top_at_mark_start(r->bottom());
163 if (top > bottom) {
164 _heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
165 }
166 r = cset->claim_next();
167 }
168 }
169
170 };
171
172 ShenandoahPartialGC::ShenandoahPartialGC(ShenandoahHeap* heap, size_t num_regions) :
173 _heap(heap),
174 _matrix(heap->connection_matrix()),
175 _root_regions(new ShenandoahHeapRegionSet(num_regions)),
176 _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())) {
177
178 assert(_matrix != NULL, "need matrix");
179
180 uint num_queues = heap->max_workers();
181 for (uint i = 0; i < num_queues; ++i) {
182 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
183 task_queue->initialize();
184 _task_queues->register_queue(i, task_queue);
185 }
186
187 from_idxs = NEW_C_HEAP_ARRAY(size_t, ShenandoahPartialInboundThreshold, mtGC);
188 }
189
190 ShenandoahPartialGC::~ShenandoahPartialGC() {
191 FREE_C_HEAP_ARRAY(size_t, from_idxs);
192 }
193
194 bool ShenandoahPartialGC::prepare() {
195 _heap->collection_set()->clear();
196 assert(_heap->collection_set()->count() == 0, "collection set not clear");
197
198 _heap->ensure_parsability(true);
199
200 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
201
202 if (UseShenandoahMatrix && PrintShenandoahMatrix) {
203 LogTarget(Info, gc) lt;
204 LogStream ls(lt);
205 _heap->connection_matrix()->print_on(&ls);
206 }
207
208 ShenandoahHeapRegionSet* regions = _heap->regions();
209 ShenandoahCollectionSet* collection_set = _heap->collection_set();
210 size_t num_regions = _heap->num_regions();
211
212 // First pass: reset all roots
213 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
214 ShenandoahHeapRegion* r = regions->get(to_idx);
215 r->set_root(false);
216 }
217
218 // Second pass: find collection set, and mark root candidates
219 _heap->shenandoahPolicy()->choose_collection_set(collection_set, true);
220
221 // Shortcut: no cset, bail
222 size_t num_cset = collection_set->count();
223
224 if (num_cset == 0) {
225 log_info(gc, ergo)("No regions with fewer inbound connections than threshold (" UINTX_FORMAT ")",
226 ShenandoahPartialInboundThreshold);
227 return false;
228 }
229
230 // Final pass: rebuild free set and region set.
231 ShenandoahFreeSet* _free_regions = _heap->free_regions();
232 _root_regions->clear();
233 _free_regions->clear();
234
235 assert(_root_regions->count() == 0, "must be cleared");
236
237 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
238 ShenandoahHeapRegion* r = regions->get(from_idx);
239 if (r->is_alloc_allowed()) {
240 _free_regions->add_region(r);
241 }
242 if (r->is_root() && !r->in_collection_set()) {
243 _root_regions->add_region(r);
244 matrix->clear_region_outbound(from_idx);
245
246 // Since root region can be allocated at, we should bound the scans
247 // in it at current top. Otherwise, one thread may evacuate objects
248 // to that root region, while another would try to scan newly evac'ed
249 // objects under the race.
250 r->set_concurrent_iteration_safe_limit(r->top());
251 }
252 }
253
254 log_info(gc,ergo)("Got "SIZE_FORMAT" collection set regions, "SIZE_FORMAT" root regions",
255 collection_set->count(), _root_regions->count());
256
257 return true;
258 }
259
260 void ShenandoahPartialGC::do_partial_collection() {
261 assert(SafepointSynchronize::is_at_safepoint(), "STW partial GC");
262 ShenandoahWorkerScope partial_gc_scope(_heap->workers(), ShenandoahWorkerPolicy::calc_workers_for_stw_partial());
263
264 _heap->set_alloc_seq_gc_start();
265
266 ShenandoahGCSession session;
267
268 GCTraceTime(Info, gc) time("Pause Partial", _heap->gc_timer(), GCCause::_no_gc, true);
269
270 if (ShenandoahVerify) {
271 _heap->verifier()->verify_before_partial();
272 }
273
274 bool has_work;
275 {
276 ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::partial_gc_prepare);
277 ShenandoahHeapLocker lock(_heap->lock());
278 has_work = prepare();
279 }
280
281 if (!has_work) {
282 reset();
283 return;
284 }
285
286 {
287 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::partial_gc_work);
288 assert(_task_queues->is_empty(), "queues must be empty after partial GC");
289
290 #if defined(COMPILER2) || INCLUDE_JVMCI
291 DerivedPointerTable::clear();
292 #endif
293
294 {
295 uint nworkers = _heap->workers()->active_workers();
296 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::partial_gc_work);
297
298 if (UseShenandoahOWST) {
299 ShenandoahTaskTerminator terminator(nworkers, task_queues());
300 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions);
301 _heap->workers()->run_task(&partial_task);
302 } else {
303 ParallelTaskTerminator terminator(nworkers, task_queues());
304 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions);
305 _heap->workers()->run_task(&partial_task);
306 }
307 }
308
309 #if defined(COMPILER2) || INCLUDE_JVMCI
310 DerivedPointerTable::update_pointers();
311 #endif
312 if (_heap->cancelled_concgc()) {
313 _task_queues->clear();
314 }
315 assert(_task_queues->is_empty(), "queues must be empty after partial GC");
316 }
317
318
319 if (! _heap->cancelled_concgc()) {
320 if (ShenandoahStringDedup::is_enabled()) {
321 ShenandoahGCPhase update_str_dedup_table(ShenandoahPhaseTimings::partial_gc_update_str_dedup_table);
322 ShenandoahStringDedup::parallel_partial_update_or_unlink();
323 }
324
325 {
326 ShenandoahGCPhase phase_recycle(ShenandoahPhaseTimings::partial_gc_recycle);
327
328 ShenandoahCollectionSet* cset = _heap->collection_set();
329
330 {
331 ShenandoahHeapLocker lock(_heap->lock());
332
333 ShenandoahPartialCollectionCleanupTask cleanup;
334 _heap->workers()->run_task(&cleanup);
335
336 // Trash everything when bitmaps are cleared.
337 cset->clear_current_index();
338 ShenandoahHeapRegion* r;
339 while((r = cset->next()) != NULL) {
340 r->make_trash();
341 }
342
343 reset();
344 }
345 }
346
347 if (ShenandoahVerify) {
348 _heap->verifier()->verify_after_partial();
349 }
350 } else {
351 // Fixup roots to make them consistent
352 _heap->fixup_roots();
353 }
354 }
355
356 void ShenandoahPartialGC::reset() {
357 _heap->collection_set()->clear();
358
359 _root_regions->clear_current_index();
360 ShenandoahHeapRegion* r;
361 while((r = _root_regions->claim_next()) != NULL) {
362 r->set_root(false);
363 }
364 _root_regions->clear();
365 }
366
367 ShenandoahObjToScanQueueSet* ShenandoahPartialGC::task_queues() {
368 return _task_queues;
369 }