1 /*
2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shared/gcTraceTime.inline.hpp"
26 #include "gc/shared/workgroup.hpp"
27 #include "gc/shared/taskqueue.inline.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
30 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp"
31 #include "gc/shenandoah/shenandoahFreeSet.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
33 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
34 #include "gc/shenandoah/shenandoahPartialGC.hpp"
35 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
36 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
37 #include "memory/iterator.hpp"
38
39 class PartialEvacuateUpdateRootsClosure : public OopClosure {
40 ShenandoahPartialGC* _partial_gc;
41 Thread* _thread;
42 SCMObjToScanQueue* _queue;
43 private:
44 template <class T>
45 void do_oop_work(T* p) { _partial_gc->process_oop<T, false>(p, _thread, _queue); }
46 public:
47 PartialEvacuateUpdateRootsClosure(SCMObjToScanQueue* q) :
48 _partial_gc(ShenandoahHeap::heap()->partial_gc()),
49 _thread(Thread::current()), _queue(q) {}
50 void do_oop(oop* p) {
51 assert(! ShenandoahHeap::heap()->is_in_reserved(p), "sanity");
52 do_oop_work(p);
53 }
54 void do_oop(narrowOop* p) { do_oop_work(p); }
55 };
56
57 class PartialEvacuateUpdateHeapClosure : public ExtendedOopClosure {
58 ShenandoahPartialGC* _partial_gc;
59 Thread* _thread;
60 SCMObjToScanQueue* _queue;
61 private:
62 template <class T>
63 void do_oop_work(T* p) {
64 _partial_gc->process_oop<T, true>(p, _thread, _queue);
65 }
66 public:
67 PartialEvacuateUpdateHeapClosure(SCMObjToScanQueue* q) :
68 _partial_gc(ShenandoahHeap::heap()->partial_gc()),
69 _thread(Thread::current()), _queue(q) {}
70 void do_oop(oop* p) { do_oop_work(p); }
71 void do_oop(narrowOop* p) { do_oop_work(p); }
72 };
73
74 class ShenandoahPartialCollectionTask : public AbstractGangTask {
75 private:
76 ShenandoahRootProcessor* _rp;
77 ParallelTaskTerminator* _terminator;
78 ShenandoahHeapRegionSet* _root_regions;
79 ShenandoahHeap* _heap;
80 public:
81 ShenandoahPartialCollectionTask(ShenandoahRootProcessor* rp,
82 ParallelTaskTerminator* terminator,
83 ShenandoahHeapRegionSet* root_regions) :
84 AbstractGangTask("Shenandoah Partial Collection"),
85 _rp(rp), _terminator(terminator), _root_regions(root_regions),
86 _heap(ShenandoahHeap::heap()) {}
87
88 void work(uint worker_id) {
89 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
90 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues();
91 SCMObjToScanQueue* q = queues->queue(worker_id);
92
93 // Step 1: Process ordinary GC roots.
94 {
95 PartialEvacuateUpdateRootsClosure roots_cl(q);
96 CLDToOopClosure cld_cl(&roots_cl);
97 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
98 _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, worker_id);
99 }
100 if (check_and_handle_cancelled_gc()) return;
101
102 PartialEvacuateUpdateHeapClosure cl(q);
103
104 // Step 2: Process all root regions.
105 {
106 ShenandoahHeapRegion* r = _root_regions->claim_next();
107 while (r != NULL) {
108 assert(r->is_root(), "must be root region");
109 _heap->marked_object_oop_iterate(r, &cl);
110 r->set_root(false);
111 if (check_and_handle_cancelled_gc()) return;
112 r = _root_regions->claim_next();
113 }
114 }
115 if (check_and_handle_cancelled_gc()) return;
116
117 // Step 3: Drain all outstanding work in queues.
118 {
119 int seed = 17;
120 SCMTask task;
121
122 uint stride = ShenandoahMarkLoopStride;
123
124 while (true) {
125 if (check_and_handle_cancelled_gc()) return;
126
127 for (uint i = 0; i < stride; i++) {
128 if ((q->pop_buffer(task) ||
129 q->pop_local(task) ||
130 q->pop_overflow(task) ||
131 queues->steal(worker_id, &seed, task))) {
132 oop obj = task.obj();
133 assert(!oopDesc::is_null(obj), "must not be null");
134 obj->oop_iterate(&cl);
135 } else {
136 if (_terminator->offer_termination()) return;
137 }
138 }
139 }
140 }
141 }
142
143 private:
144 bool check_and_handle_cancelled_gc() {
145 if (_heap->cancelled_concgc()) {
146 ShenandoahCancelledTerminatorTerminator tt;
147 while (! _terminator->offer_termination(&tt));
148 return true;
149 }
150 return false;
151 }
152 };
153
154 ShenandoahPartialGC::ShenandoahPartialGC(ShenandoahHeap* heap, size_t max_regions) :
155 _heap(heap),
156 _matrix(heap->connection_matrix()),
157 _root_regions(new ShenandoahHeapRegionSet(max_regions)),
158 _task_queues(new SCMObjToScanQueueSet(heap->max_workers())) {
159
160 assert(_matrix != NULL, "need matrix");
161
162 uint num_queues = heap->max_workers();
163 for (uint i = 0; i < num_queues; ++i) {
164 SCMObjToScanQueue* task_queue = new SCMObjToScanQueue();
165 task_queue->initialize();
166 _task_queues->register_queue(i, task_queue);
167 }
168
169 }
170
171 void ShenandoahPartialGC::prepare() {
172
173 if (ShenandoahVerify || VerifyShenandoahMatrix) {
174 log_debug(gc, ergo)("Verifying before partial collection");
175 _heap->verify_heap_reachable_at_safepoint();
176 log_debug(gc, ergo)("Verification passed");
177 }
178
179 _heap->collection_set()->clear();
180 assert(_heap->collection_set()->count() == 0, "collection set not clear");
181
182 _heap->ensure_parsability(true);
183
184 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
185 ShenandoahHeapRegionSet* regions = _heap->regions();
186 ShenandoahCollectionSet* collection_set = _heap->collection_set();
187 ShenandoahFreeSet* free_set = _heap->free_regions();
188 free_set->clear();
189 _root_regions->clear();
190 assert(_root_regions->count() == 0, "must be cleared");
191 size_t num_regions = _heap->num_regions();
192
193 // First pass: find collection set.
194 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
195 ShenandoahHeapRegion* region = regions->get(to_idx);
196 region->set_root(false);
197 guarantee(! region->is_root(), "must not be root here");
198 if (region->is_humongous() || region->is_empty() || region->is_pinned()) continue;
199 assert(! _heap->region_in_collection_set(to_idx), "must not be in cset yet");
200 uint num_incoming = 0;
201 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
202 if (matrix->is_connected(from_idx, to_idx)) {
203 num_incoming++;
204 }
205 }
206 if (num_incoming < ShenandoahPartialInboundThreshold) {
207 collection_set->add_region(region);
208 _heap->set_region_in_collection_set(to_idx, true);
209 }
210 }
211
212 if (UseShenandoahMatrix && PrintShenandoahMatrix) {
213 outputStream* log = Log(gc)::info_stream();
214 matrix->print_on(log);
215 }
216
217 // Second pass: find all root regions.
218 size_t num_cset = collection_set->count();
219 for (uint idx = 0; idx < num_cset; idx++) {
220 ShenandoahHeapRegion* region = collection_set->get(idx);
221 size_t to_idx = region->region_number();
222 assert(! region->is_humongous() && ! region->is_empty() && ! region->is_pinned(), "sanity");
223 assert(_heap->region_in_collection_set(to_idx), "more sanity");
224 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
225 if (matrix->is_connected(from_idx, to_idx)) {
226 ShenandoahHeapRegion* r = regions->get(from_idx);
227 if (! r->is_root() && ! _heap->region_in_collection_set(from_idx)) {
228 _root_regions->add_region(r);
229 r->set_root(true);
230 }
231 }
232 }
233 }
234 // Final pass: free regions.
235 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
236 ShenandoahHeapRegion* region = regions->get(to_idx);
237 if (! region->is_humongous() &&
238 ! region->is_pinned() &&
239 ! region->is_root() &&
240 ! _heap->in_collection_set(region)) {
241
242 free_set->add_region(region);
243 }
244 if (region->is_root()) {
245 // We rebuild the outbound matrix for all roots, thus we can clear them before.
246 // This makes the matrix more precise.
247 matrix->clear_region_outbound(region->region_number());
248 }
249 }
250 log_debug(gc, ergo)("got "SIZE_FORMAT" cset regions", collection_set->count());
251 log_debug(gc, ergo)("got "SIZE_FORMAT" root regions", _root_regions->count());
252 }
253
254 void ShenandoahPartialGC::do_partial_collection() {
255
256 _heap->gc_timer()->register_gc_start();
257 ShenandoahCollectorPolicy* policy = _heap->shenandoahPolicy();
258 policy->record_phase_start(ShenandoahCollectorPolicy::partial_gc);
259
260 {
261 GCTraceTime(Info, gc) time("Pause Partial", _heap->gc_timer(), GCCause::_no_gc, true);
262
263 assert(_task_queues->is_empty(), "queues must be empty after partial GC");
264
265 COMPILER2_PRESENT(DerivedPointerTable::clear());
266
267 {
268 ClassLoaderDataGraph::clear_claimed_marks();
269 uint nworkers = _heap->workers()->active_workers();
270 ShenandoahRootProcessor rp(_heap, nworkers);
271
272 if (UseShenandoahOWST) {
273 ShenandoahTaskTerminator terminator(nworkers, task_queues());
274 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions);
275 _heap->workers()->run_task(&partial_task);
276 } else {
277 ParallelTaskTerminator terminator(nworkers, task_queues());
278 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions);
279 _heap->workers()->run_task(&partial_task);
280 }
281 }
282
283 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
284
285 if (_heap->cancelled_concgc()) {
286 _task_queues->clear();
287 }
288 assert(_task_queues->is_empty(), "queues must be empty after partial GC");
289
290 if (! _heap->cancelled_concgc()) {
291 if (ShenandoahVerify || VerifyShenandoahMatrix) {
292 log_debug(gc, ergo)("Verifying after partial collection");
293 _heap->verify_heap_reachable_at_safepoint();
294 log_debug(gc, ergo)("Verification passed");
295 }
296
297 ShenandoahHeap::ShenandoahHeapLock heap_lock(_heap);
298 size_t num_cset = _heap->collection_set()->count();
299 for (size_t i = 0; i < num_cset; i++) {
300 ShenandoahHeapRegion* r = _heap->collection_set()->get(i);
301 _heap->decrease_used(r->used());
302 HeapWord* bottom = r->bottom();
303 HeapWord* top = _heap->complete_top_at_mark_start(r->bottom());
304 if (top > bottom) {
305 _heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
306 }
307 r->recycle();
308 _heap->free_regions()->add_region(r);
309 }
310
311 reset();
312 }
313 }
314
315 policy->record_phase_end(ShenandoahCollectorPolicy::partial_gc);
316 _heap->gc_timer()->register_gc_end();
317 }
318
319 void ShenandoahPartialGC::reset() {
320 _heap->collection_set()->clear();
321 _heap->clear_cset_fast_test();
322 _root_regions->clear();
323 }
324
325 template <class T, bool UPDATE_MATRIX>
326 void ShenandoahPartialGC::process_oop(T* p, Thread* thread, SCMObjToScanQueue* queue) {
327 T o = oopDesc::load_heap_oop(p);
328 if (! oopDesc::is_null(o)) {
329 oop obj = oopDesc::decode_heap_oop_not_null(o);
330 if (_heap->in_collection_set(obj)) {
331 oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
332 if (oopDesc::unsafe_equals(obj, forw)) {
333 bool evacuated = false;
334 forw = _heap->evacuate_object(obj, thread, evacuated);
335
336 // Only the thread that succeeded evacuating this object pushes it to its work queue.
337 if (evacuated) {
338 assert(forw->is_oop(), "sanity");
339 bool succeeded = queue->push(SCMTask(forw));
340 assert(succeeded, "must succeed to push to task queue");
341 }
342 }
343 assert(! oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must be evacuated");
344 // Update reference.
345 oopDesc::encode_store_heap_oop_not_null(p, forw);
346 obj = forw; // For matrix update below.
347 }
348 if (UPDATE_MATRIX) {
349 #ifdef ASSERT
350 oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
351 assert(oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must not be evacuated: "PTR_FORMAT" -> "PTR_FORMAT, p2i(obj), p2i(forw));
352 #endif
353 _matrix->set_connected(p, obj);
354 }
355 }
356 }
357
358 SCMObjToScanQueueSet* ShenandoahPartialGC::task_queues() {
359 return _task_queues;
360 }