1 /*
2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "gc/shared/gcTraceTime.inline.hpp"
25 #include "gc/shared/workgroup.hpp"
26 #include "gc/shared/taskqueue.inline.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
29 #include "gc/shenandoah/shenandoahFreeSet.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahPartialGC.hpp"
33 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
34 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
35 #include "memory/iterator.hpp"
36
37 class PartialEvacuateUpdateRootsClosure : public OopClosure {
38 ShenandoahPartialGC* _partial_gc;
39 Thread* _thread;
40 SCMObjToScanQueue* _queue;
41 private:
42 template <class T>
43 void do_oop_work(T* p) { _partial_gc->process_oop(p, _thread, _queue, false); }
44 public:
45 PartialEvacuateUpdateRootsClosure(SCMObjToScanQueue* q) :
46 _partial_gc(ShenandoahHeap::heap()->partial_gc()),
47 _thread(Thread::current()), _queue(q) {}
48 void do_oop(oop* p) {
49 assert(! ShenandoahHeap::heap()->is_in_reserved(p), "sanity");
50 do_oop_work(p);
51 }
52 void do_oop(narrowOop* p) { do_oop_work(p); }
53 };
54
55 class PartialEvacuateUpdateHeapClosure : public ExtendedOopClosure {
56 ShenandoahPartialGC* _partial_gc;
57 Thread* _thread;
58 SCMObjToScanQueue* _queue;
59 private:
60 template <class T>
61 void do_oop_work(T* p) {
62 _partial_gc->process_oop(p, _thread, _queue, true);
63 }
64 public:
65 PartialEvacuateUpdateHeapClosure(SCMObjToScanQueue* q) :
66 _partial_gc(ShenandoahHeap::heap()->partial_gc()),
67 _thread(Thread::current()), _queue(q) {}
68 void do_oop(oop* p) { do_oop_work(p); }
69 void do_oop(narrowOop* p) { do_oop_work(p); }
70 };
71
72 class ShenandoahPartialCollectionTask : public AbstractGangTask {
73 private:
74 ShenandoahRootProcessor* _rp;
75 ShenandoahHeapRegionSet* _root_regions;
76 ShenandoahHeap* _heap;
77 public:
78 ShenandoahPartialCollectionTask(ShenandoahRootProcessor* rp,
79 ShenandoahHeapRegionSet* root_regions) :
80 AbstractGangTask("Shenandoah Partial Collection"),
81 _rp(rp), _root_regions(root_regions),
82 _heap(ShenandoahHeap::heap()) {}
83
84 void work(uint worker_id) {
85 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
86 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues();
87 SCMObjToScanQueue* q = queues->queue(worker_id);
88 {
89 // First process ordinary GC roots.
90 PartialEvacuateUpdateRootsClosure roots_cl(q);
91 CLDToOopClosure cld_cl(&roots_cl);
92 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
93 _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, worker_id);
94 }
95 drain_queue(worker_id);
96 if (_heap->cancelled_concgc()) { q->set_empty(); return; }
97 {
98 // Then process root regions.
99 PartialEvacuateUpdateHeapClosure cl(q);
100 ShenandoahHeapRegion* r = _root_regions->claim_next();
101 while (r != NULL) {
102 // We rebuild the outbound matrix for all roots, thus we can clear them before.
103 // This makes the matrix more precise.
104 matrix->clear_region_outbound(r->region_number());
105
106 assert(r->is_root(), "must be root region");
107 r->oop_iterate(&cl);
108 r->set_root(false);
109 drain_queue(worker_id);
110 if (_heap->cancelled_concgc()) { q->set_empty(); return; }
111 r = _root_regions->claim_next();
112 }
113 }
114 }
115
116 void drain_queue(uint worker_id) {
117 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues();
118 SCMObjToScanQueue* q = queues->queue(worker_id);
119 PartialEvacuateUpdateHeapClosure cl(q);
120 // Empty queue if necessary.
121 int seed = 17;
122 SCMTask task;
123 while ((q->pop_buffer(task) ||
124 q->pop_local(task) ||
125 q->pop_overflow(task)) &&
126 !_heap->cancelled_concgc()) {
127 oop obj = task.obj();
128 assert(! oopDesc::is_null(obj), "must not be null");
129 obj->oop_iterate(&cl);
130 }
131 }
132
133 };
134
135 ShenandoahPartialGC::ShenandoahPartialGC(ShenandoahHeap* heap, uint max_regions) :
136 _heap(heap),
137 _root_regions(new ShenandoahHeapRegionSet(max_regions)),
138 _task_queues(new SCMObjToScanQueueSet(heap->max_workers())) {
139
140 uint num_queues = heap->max_workers();
141 for (uint i = 0; i < num_queues; ++i) {
142 SCMObjToScanQueue* task_queue = new SCMObjToScanQueue();
143 task_queue->initialize();
144 _task_queues->register_queue(i, task_queue);
145 }
146
147 }
148
149 ShenandoahHeapRegionSet* ShenandoahPartialGC::root_regions() {
150 return _root_regions;
151 }
152
153 void ShenandoahPartialGC::prepare() {
154 _heap->collection_set()->clear();
155 assert(_heap->collection_set()->count() == 0, "collection set not clear");
156
157 _heap->ensure_parsability(true);
158
159 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
160 ShenandoahHeapRegionSet* regions = _heap->regions();
161 ShenandoahCollectionSet* collection_set = _heap->collection_set();
162 ShenandoahFreeSet* free_set = _heap->free_regions();
163 free_set->clear();
164 _root_regions->clear();
165 assert(_root_regions->count() == 0, "must be cleared");
166 uint num_regions = _heap->num_regions();
167
168 // First pass: find collection set.
169 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
170 ShenandoahHeapRegion* region = regions->get(to_idx);
171 region->set_root(false);
172 guarantee(! region->is_root(), "must not be root here");
173 if (region->is_humongous() || region->is_empty() || region->is_pinned()) continue;
174 assert(! _heap->region_in_collection_set(to_idx), "must not be in cset yet");
175 uint num_incoming = 0;
176 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
177 if (matrix->is_connected(from_idx, to_idx)) {
178 num_incoming++;
179 }
180 }
181 if (num_incoming < ShenandoahPartialInboundThreshold) {
182 collection_set->add_region(region);
183 _heap->set_region_in_collection_set(to_idx, true);
184 }
185 }
186 // Second pass: find all root regions.
187 uint num_cset = collection_set->count();
188 for (uint idx = 0; idx < num_cset; idx++) {
189 ShenandoahHeapRegion* region = collection_set->get(idx);
190 uint to_idx = region->region_number();
191 assert(! region->is_humongous() && ! region->is_empty() && ! region->is_pinned(), "sanity");
192 assert(_heap->region_in_collection_set(to_idx), "more sanity");
193 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
194 if (matrix->is_connected(from_idx, to_idx)) {
195 ShenandoahHeapRegion* r = regions->get(from_idx);
196 if (! r->is_root() && ! _heap->region_in_collection_set(from_idx)) {
197 _root_regions->add_region(r);
198 r->set_root(true);
199 }
200 }
201 }
202 }
203 // Final pass: free regions.
204 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
205 ShenandoahHeapRegion* region = regions->get(to_idx);
206 if (! region->is_humongous() &&
207 ! region->is_pinned() &&
208 ! region->is_root() &&
209 ! _heap->in_collection_set(region)) {
210
211 free_set->add_region(region);
212 }
213 }
214 log_debug(gc, ergo)("got "SIZE_FORMAT" cset regions", collection_set->count());
215 log_debug(gc, ergo)("got "SIZE_FORMAT" root regions", _root_regions->count());
216 }
217
218 void ShenandoahPartialGC::do_partial_collection() {
219
220 _heap->gc_timer()->register_gc_start();
221 ShenandoahCollectorPolicy* policy = _heap->shenandoahPolicy();
222 policy->record_phase_start(ShenandoahCollectorPolicy::partial_gc);
223
224 {
225 GCTraceTime(Info, gc) time("Pause Partial", _heap->gc_timer(), GCCause::_no_gc, true);
226
227 if (ShenandoahVerify || VerifyShenandoahMatrix) {
228 log_debug(gc, ergo)("Verifying before partial collection");
229 _heap->verify_heap_reachable_at_safepoint();
230 log_debug(gc, ergo)("Verification passed");
231 }
232
233 COMPILER2_PRESENT(DerivedPointerTable::clear());
234
235 {
236 ClassLoaderDataGraph::clear_claimed_marks();
237 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers());
238 ShenandoahPartialCollectionTask partial_task(&rp, _root_regions);
239 _heap->workers()->run_task(&partial_task);
240 }
241
242 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
243
244 if (! _heap->cancelled_concgc()) {
245 if (ShenandoahVerify || VerifyShenandoahMatrix) {
246 log_debug(gc, ergo)("Verifying after partial collection");
247 _heap->verify_heap_reachable_at_safepoint();
248 log_debug(gc, ergo)("Verification passed");
249 }
250
251 ShenandoahHeap::ShenandoahHeapLock heap_lock(_heap);
252 uint num_cset = _heap->collection_set()->count();
253 for (uint i = 0; i < num_cset; i++) {
254 ShenandoahHeapRegion* r = _heap->collection_set()->get(i);
255 _heap->decrease_used(r->used());
256 r->recycle();
257 _heap->free_regions()->add_region(r);
258 }
259
260 reset();
261 }
262 }
263
264 policy->record_phase_end(ShenandoahCollectorPolicy::partial_gc);
265 _heap->gc_timer()->register_gc_end();
266 }
267
268 void ShenandoahPartialGC::reset() {
269 _heap->collection_set()->clear();
270 _heap->clear_cset_fast_test();
271 _root_regions->clear();
272 }
273
274 template <class T>
275 void ShenandoahPartialGC::process_oop(T* p, Thread* thread, SCMObjToScanQueue* queue, bool update_matrix) {
276 T o = oopDesc::load_heap_oop(p);
277 if (! oopDesc::is_null(o)) {
278 oop obj = oopDesc::decode_heap_oop_not_null(o);
279 if (_heap->in_collection_set(obj)) {
280 oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
281 if (oopDesc::unsafe_equals(obj, forw)) {
282 forw = _heap->evacuate_object(obj, thread);
283 }
284 assert(! oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must be evacuated");
285 // TODO: Only the thread that succeeds *evacuating* the object should need to
286 // update the matrix and push the evacuated object to its queue. This would also
287 // enable to only have one CAS (the one in evacuate_object()) and use simple
288 // store for updating the ref.
289 oop oldval = _heap->atomic_compare_exchange_oop(forw, p, obj);
290 if (oopDesc::unsafe_equals(obj, oldval)) {
291 assert(forw->is_oop(), "sanity");
292 bool succeeded = queue->push(SCMTask(forw));
293 assert(succeeded, "must succeed to push to task queue");
294 } else {
295 assert(oopDesc::unsafe_equals(oldval, forw), "other thread must have punched the same forwarded oop");
296 }
297 obj = forw; // For matrix update below.
298 }
299 // TODO: Make this templated
300 if (update_matrix) {
301 #ifdef ASSERT
302 oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
303 assert(oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must not be evacuated");
304 #endif
305 uint from_idx = _heap->heap_region_index_containing(p);
306 uint to_idx = _heap->heap_region_index_containing(obj);
307 _heap->connection_matrix()->set_connected(from_idx, to_idx, true);
308 }
309 }
310 }
311
312 bool ShenandoahPartialGC::is_in_root_region(oop obj) {
313 // TODO: make this very fast!!
314 ShenandoahHeapRegion* r = _heap->heap_region_containing(obj);
315 return _root_regions->contains(r);
316 }
317
318 SCMObjToScanQueueSet* ShenandoahPartialGC::task_queues() {
319 return _task_queues;
320 }