1 /*
2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "gc/shared/gcTraceTime.inline.hpp"
25 #include "gc/shared/workgroup.hpp"
26 #include "gc/shared/taskqueue.inline.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
29 #include "gc/shenandoah/shenandoahFreeSet.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahPartialGC.hpp"
33 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
34 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
35 #include "memory/iterator.hpp"
36
37 class PartialEvacuateUpdateRootsClosure : public OopClosure {
38 ShenandoahPartialGC* _partial_gc;
39 Thread* _thread;
40 SCMObjToScanQueue* _queue;
41 private:
42 template <class T>
43 void do_oop_work(T* p) { _partial_gc->process_oop(p, _thread, _queue, false); }
44 public:
45 PartialEvacuateUpdateRootsClosure(SCMObjToScanQueue* q) :
46 _partial_gc(ShenandoahHeap::heap()->partial_gc()),
47 _thread(Thread::current()), _queue(q) {}
48 void do_oop(oop* p) { do_oop_work(p); }
49 void do_oop(narrowOop* p) { do_oop_work(p); }
50 };
51
52 class PartialEvacuateUpdateHeapClosure : public ExtendedOopClosure {
53 ShenandoahPartialGC* _partial_gc;
54 Thread* _thread;
55 SCMObjToScanQueue* _queue;
56 private:
57 template <class T>
58 void do_oop_work(T* p) {
59 _partial_gc->process_oop(p, _thread, _queue, true);
60 }
61 public:
62 PartialEvacuateUpdateHeapClosure(SCMObjToScanQueue* q) :
63 _partial_gc(ShenandoahHeap::heap()->partial_gc()),
64 _thread(Thread::current()), _queue(q) {}
65 void do_oop(oop* p) { do_oop_work(p); }
66 void do_oop(narrowOop* p) { do_oop_work(p); }
67 };
68
69 class ShenandoahPartialCollectionTask : public AbstractGangTask {
70 private:
71 ShenandoahRootProcessor* _rp;
72 ShenandoahHeapRegionSet* _root_regions;
73 ShenandoahHeap* _heap;
74 public:
75 ShenandoahPartialCollectionTask(ShenandoahRootProcessor* rp,
76 ShenandoahHeapRegionSet* root_regions) :
77 AbstractGangTask("Shenandoah Partial Collection"),
78 _rp(rp), _root_regions(root_regions),
79 _heap(ShenandoahHeap::heap()) {}
80
81 void work(uint worker_id) {
82 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues();
83 SCMObjToScanQueue* q = queues->queue(worker_id);
84 {
85 // First process ordinary GC roots.
86 PartialEvacuateUpdateRootsClosure roots_cl(q);
87 CLDToOopClosure cld_cl(&roots_cl);
88 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
89 _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, worker_id);
90 }
91 drain_queue(worker_id);
92 if (_heap->cancelled_concgc()) { q->set_empty(); return; }
93 {
94 // Then process root regions.
95 PartialEvacuateUpdateHeapClosure cl(q);
96 ShenandoahHeapRegion* r = _root_regions->claim_next();
97 while (r != NULL) {
98 r->oop_iterate(&cl);
99 drain_queue(worker_id);
100 if (_heap->cancelled_concgc()) { q->set_empty(); return; }
101 r = _root_regions->claim_next();
102 }
103 }
104 }
105
106 void drain_queue(uint worker_id) {
107 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues();
108 SCMObjToScanQueue* q = queues->queue(worker_id);
109 PartialEvacuateUpdateHeapClosure cl(q);
110 // Empty queue if necessary.
111 int seed = 17;
112 SCMTask task;
113 while ((q->pop_buffer(task) ||
114 q->pop_local(task) ||
115 q->pop_overflow(task)) &&
116 !_heap->cancelled_concgc()) {
117 oop obj = task.obj();
118 assert(! oopDesc::is_null(obj), "must not be null");
119 obj->oop_iterate(&cl);
120 }
121 }
122
123 };
124
125 ShenandoahPartialGC::ShenandoahPartialGC(ShenandoahHeap* heap, uint max_regions) :
126 _heap(heap),
127 _root_regions(new ShenandoahHeapRegionSet(max_regions)),
128 _task_queues(new SCMObjToScanQueueSet(heap->max_workers())) {
129
130 uint num_queues = heap->max_workers();
131 for (uint i = 0; i < num_queues; ++i) {
132 SCMObjToScanQueue* task_queue = new SCMObjToScanQueue();
133 task_queue->initialize();
134 _task_queues->register_queue(i, task_queue);
135 }
136
137 }
138
139 ShenandoahHeapRegionSet* ShenandoahPartialGC::root_regions() {
140 return _root_regions;
141 }
142
143 void ShenandoahPartialGC::prepare() {
144 _heap->collection_set()->clear();
145 assert(_heap->collection_set()->count() == 0, "collection set not clear");
146
147 _heap->ensure_parsability(true);
148
149 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix();
150 ShenandoahHeapRegionSet* regions = _heap->regions();
151 ShenandoahCollectionSet* collection_set = _heap->collection_set();
152 ShenandoahFreeSet* free_set = _heap->free_regions();
153 free_set->clear();
154 _root_regions->clear();
155 assert(_root_regions->count() == 0, "must be cleared");
156 uint num_regions = _heap->num_regions();
157
158 // First pass: find collection set.
159 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
160 ShenandoahHeapRegion* region = regions->get(to_idx);
161 if (region->is_humongous() || region->is_empty() || region->is_pinned()) continue;
162 assert(! _heap->region_in_collection_set(to_idx), "must not be in cset yet");
163 uint num_incoming = 0;
164 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
165 if (matrix->is_connected(from_idx, to_idx)) {
166 num_incoming++;
167 }
168 }
169 if (num_incoming < ShenandoahPartialInboundThreshold) {
170 collection_set->add_region(region);
171 _heap->set_region_in_collection_set(to_idx, true);
172 }
173 }
174 // Second pass: find all root regions.
175 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
176 ShenandoahHeapRegion* region = regions->get(to_idx);
177 if (region->is_humongous() || region->is_empty() || region->is_pinned()) continue;
178 if (_heap->region_in_collection_set(to_idx)) {
179 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
180 if (matrix->is_connected(from_idx, to_idx)) {
181 ShenandoahHeapRegion* r = regions->get(from_idx);
182 if (! _root_regions->contains(r)) {
183 _root_regions->add_region(r);
184 }
185 }
186 }
187 }
188 }
189 // Final pass: free regions.
190 for (uint to_idx = 0; to_idx < num_regions; to_idx++) {
191 ShenandoahHeapRegion* region = regions->get(to_idx);
192 if (! region->is_humongous() &&
193 ! region->is_pinned() &&
194 ! _root_regions->contains(region) &&
195 ! _heap->in_collection_set(region)) {
196
197 free_set->add_region(region);
198 }
199 }
200 log_debug(gc, ergo)("got "SIZE_FORMAT" cset regions", collection_set->count());
201 log_debug(gc, ergo)("got "SIZE_FORMAT" root regions", _root_regions->count());
202 }
203
204 void ShenandoahPartialGC::do_partial_collection() {
205
206 _heap->gc_timer()->register_gc_start();
207 {
208 GCTraceTime(Info, gc) time("Pause Partial", _heap->gc_timer(), GCCause::_no_gc, true);
209
210 COMPILER2_PRESENT(DerivedPointerTable::clear());
211
212 {
213 ClassLoaderDataGraph::clear_claimed_marks();
214 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers());
215 ShenandoahPartialCollectionTask partial_task(&rp, _root_regions);
216 _heap->workers()->run_task(&partial_task);
217 }
218
219 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
220
221 if (! _heap->cancelled_concgc()) {
222 ShenandoahHeap::ShenandoahHeapLock heap_lock(_heap);
223 uint num_cset = _heap->collection_set()->count();
224 for (uint i = 0; i < num_cset; i++) {
225 ShenandoahHeapRegion* r = _heap->collection_set()->get(i);
226 _heap->decrease_used(r->used());
227 r->recycle();
228 _heap->free_regions()->add_region(r);
229 }
230
231 reset();
232 }
233 }
234 _heap->gc_timer()->register_gc_end();
235 }
236
237 void ShenandoahPartialGC::reset() {
238 _heap->collection_set()->clear();
239 _heap->clear_cset_fast_test();
240 _root_regions->clear();
241 }
242
243 template <class T>
244 void ShenandoahPartialGC::process_oop(T* p, Thread* thread, SCMObjToScanQueue* queue, bool update_matrix) {
245 T o = oopDesc::load_heap_oop(p);
246 if (! oopDesc::is_null(o)) {
247 oop obj = oopDesc::decode_heap_oop_not_null(o);
248 if (_heap->in_collection_set(obj)) {
249 oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
250 if (oopDesc::unsafe_equals(obj, forw)) {
251 forw = _heap->evacuate_object(obj, thread);
252 }
253 assert(! oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must be evacuated");
254 // oopDesc::encode_store_heap_oop_not_null(p, forw);
255 // queue->push(forw);
256 // Can use simple store for GC roots.
257 if (oopDesc::unsafe_equals(obj, _heap->atomic_compare_exchange_oop(forw, p, obj))) {
258 if (update_matrix) {
259 uint from_idx = _heap->heap_region_index_containing(p);
260 uint to_idx = _heap->heap_region_index_containing(forw);
261 _heap->connection_matrix()->set_connected(from_idx, to_idx, true);
262 }
263 assert(forw->is_oop(), "sanity");
264 bool succeeded = queue->push(SCMTask(forw));
265 assert(succeeded, "must succeed to push to task queue");
266 }
267 }
268 }
269 }
270
271 bool ShenandoahPartialGC::is_in_root_region(oop obj) {
272 // TODO: make this very fast!!
273 ShenandoahHeapRegion* r = _heap->heap_region_containing(obj);
274 return _root_regions->contains(r);
275 }
276
277 SCMObjToScanQueueSet* ShenandoahPartialGC::task_queues() {
278 return _task_queues;
279 }