1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shared/gcTraceTime.inline.hpp"
27 #include "gc/shared/markBitMap.inline.hpp"
28 #include "gc/shared/workgroup.hpp"
29 #include "gc/shared/taskqueue.inline.hpp"
30 #include "gc/shared/weakProcessor.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahFreeSet.hpp"
35 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
37 #include "gc/shenandoah/shenandoahHeap.hpp"
38 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
40 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
41 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
42 #include "gc/shenandoah/shenandoahStringDedup.hpp"
43 #include "gc/shenandoah/shenandoahStrDedupQueue.inline.hpp"
44 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
45 #include "gc/shenandoah/shenandoahUtils.hpp"
46 #include "gc/shenandoah/shenandoahVerifier.hpp"
47 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
48 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
49
50 #include "memory/iterator.hpp"
51
52 /**
53 * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
54 * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
55 * is incremental-update-based.
56 *
57 * NOTE on interaction with TAMS: we want to avoid traversing new objects for
58 * several reasons:
59 * - We will not reclaim them in this cycle anyway, because they are not in the
60 * cset
61 * - It makes up for the bulk of work during final-pause
62 * - It also shortens the concurrent cycle because we don't need to
63 * pointlessly traverse through newly allocated objects.
64 * - As a nice side-effect, it solves the I-U termination problem (mutators
65 * cannot outrun the GC by allocating like crazy)
66 * - It is an easy way to achieve MWF. What MWF does is to also enqueue the
67 * target object of stores if it's new. Treating new objects live implicitely
68 * achieves the same, but without extra barriers. I think the effect of
69 * shortened final-pause (mentioned above) is the main advantage of MWF. In
70 * particular, we will not see the head of a completely new long linked list
71 * in final-pause and end up traversing huge chunks of the heap there.
72 * - We don't need to see/update the fields of new objects either, because they
73 * are either still null, or anything that's been stored into them has been
74 * evacuated+enqueued before (and will thus be treated later).
75 *
76 * We achieve this by setting TAMS for each region, and everything allocated
77 * beyond TAMS will be 'implicitely marked'.
78 *
79 * Gotchas:
80 * - While we want new objects to be implicitely marked, we don't want to count
81 * them alive. Otherwise the next cycle wouldn't pick them up and consider
82 * them for cset. This means that we need to protect such regions from
83 * getting accidentally thrashed at the end of traversal cycle. This is why I
84 * keep track of alloc-regions and check is_alloc_region() in the trashing
85 * code.
86 * - We *need* to traverse through evacuated objects. Those objects are
87 * pre-existing, and any references in them point to interesting objects that
88 * we need to see. We also want to count them as live, because we just
89 * determined that they are alive :-) I achieve this by upping TAMS
90 * concurrently for every gclab/gc-shared alloc before publishing the
91 * evacuated object. This way, the GC threads will not consider such objects
92 * implictely marked, and traverse through them as normal.
93 */
94 class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure {
95 private:
96 ShenandoahObjToScanQueue* _queue;
97 ShenandoahTraversalGC* _traversal_gc;
98 ShenandoahHeap* _heap;
99 public:
100 ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) :
101 _queue(q), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
102 _heap(ShenandoahHeap::heap())
103 { }
104
105 void do_buffer(void** buffer, size_t size) {
106 for (size_t i = 0; i < size; ++i) {
107 oop* p = (oop*) &buffer[i];
108 oop obj = RawAccess<>::oop_load(p);
109 shenandoah_assert_not_forwarded(p, obj);
110 if (!_heap->is_marked_next(obj) && _heap->mark_next(obj)) {
111 _queue->push(ShenandoahMarkTask(obj));
112 }
113 }
114 }
115 };
116
117 class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure {
118 ShenandoahTraversalSATBBufferClosure* _satb_cl;
119
120 public:
121 ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) :
122 _satb_cl(satb_cl) {}
123
124 void do_thread(Thread* thread) {
125 if (thread->is_Java_thread()) {
126 JavaThread* jt = (JavaThread*)thread;
127 ShenandoahThreadLocalData::satb_mark_queue(jt).apply_closure_and_empty(_satb_cl);
128 } else if (thread->is_VM_thread()) {
129 ShenandoahBarrierSet::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
130 }
131 }
132 };
133
134 // Like CLDToOopClosure, but clears has_modified_oops, so that we can record modified CLDs during traversal
135 // and remark them later during final-traversal.
136 class ShenandoahMarkCLDClosure : public CLDClosure {
137 private:
138 OopClosure* _cl;
139 public:
140 ShenandoahMarkCLDClosure(OopClosure* cl) : _cl(cl) {}
141 void do_cld(ClassLoaderData* cld) {
142 cld->oops_do(_cl, true, true);
143 }
144 };
145
146 // Like CLDToOopClosure, but only process modified CLDs
147 class ShenandoahRemarkCLDClosure : public CLDClosure {
148 private:
149 OopClosure* _cl;
150 public:
151 ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {}
152 void do_cld(ClassLoaderData* cld) {
153 if (cld->has_modified_oops()) {
154 cld->oops_do(_cl, true, true);
155 }
156 }
157 };
158
159 class ShenandoahInitTraversalCollectionTask : public AbstractGangTask {
160 private:
161 ShenandoahRootProcessor* _rp;
162 ShenandoahHeap* _heap;
163 public:
164 ShenandoahInitTraversalCollectionTask(ShenandoahRootProcessor* rp) :
165 AbstractGangTask("Shenandoah Init Traversal Collection"),
166 _rp(rp),
167 _heap(ShenandoahHeap::heap()) {}
168
169 void work(uint worker_id) {
170 ShenandoahEvacOOMScope oom_evac_scope;
171 ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
172 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
173
174 bool process_refs = _heap->process_references();
175 bool unload_classes = _heap->unload_classes();
176 ReferenceProcessor* rp = NULL;
177 if (process_refs) {
178 rp = _heap->ref_processor();
179 }
180
181 // Step 1: Process ordinary GC roots.
182 {
183 ShenandoahTraversalClosure roots_cl(q, rp);
184 ShenandoahMarkCLDClosure cld_cl(&roots_cl);
185 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
186 if (unload_classes) {
187 _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, &code_cl, NULL, worker_id);
188 } else {
189 _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, NULL, worker_id);
190 }
191 }
192 }
193 };
194
195 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
196 private:
197 ParallelTaskTerminator* _terminator;
198 ShenandoahHeap* _heap;
199 public:
200 ShenandoahConcurrentTraversalCollectionTask(ParallelTaskTerminator* terminator) :
201 AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
202 _terminator(terminator),
203 _heap(ShenandoahHeap::heap()) {}
204
205 void work(uint worker_id) {
206 ShenandoahEvacOOMScope oom_evac_scope;
207 ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
208 ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
209 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
210
211 // Drain all outstanding work in queues.
212 traversal_gc->main_loop(worker_id, _terminator, true);
213 }
214 };
215
216 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
217 private:
218 ShenandoahRootProcessor* _rp;
219 ParallelTaskTerminator* _terminator;
220 ShenandoahHeap* _heap;
221 public:
222 ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ParallelTaskTerminator* terminator) :
223 AbstractGangTask("Shenandoah Final Traversal Collection"),
224 _rp(rp),
225 _terminator(terminator),
226 _heap(ShenandoahHeap::heap()) {}
227
228 void work(uint worker_id) {
229 ShenandoahEvacOOMScope oom_evac_scope;
230 ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
231
232 ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
233 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
234
235 bool process_refs = _heap->process_references();
236 bool unload_classes = _heap->unload_classes();
237 ReferenceProcessor* rp = NULL;
238 if (process_refs) {
239 rp = _heap->ref_processor();
240 }
241
242 // Step 1: Drain outstanding SATB queues.
243 // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
244 ShenandoahTraversalSATBBufferClosure satb_cl(q);
245 {
246 // Process remaining finished SATB buffers.
247 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
248 while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
249 // Process remaining threads SATB buffers below.
250 }
251
252 // Step 1: Process ordinary GC roots.
253 if (!_heap->is_degenerated_gc_in_progress()) {
254 ShenandoahTraversalClosure roots_cl(q, rp);
255 CLDToOopClosure cld_cl(&roots_cl);
256 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
257 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
258 if (unload_classes) {
259 ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
260 _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, &code_cl, &tc, worker_id);
261 } else {
262 _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, &tc, worker_id);
263 }
264 } else {
265 ShenandoahTraversalDegenClosure roots_cl(q, rp);
266 CLDToOopClosure cld_cl(&roots_cl);
267 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
268 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
269 if (unload_classes) {
270 ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
271 _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, &code_cl, &tc, worker_id);
272 } else {
273 _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, &tc, worker_id);
274 }
275 }
276
277 {
278 ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
279 ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
280
281 // Step 3: Finally drain all outstanding work in queues.
282 traversal_gc->main_loop(worker_id, _terminator, false);
283 }
284
285 }
286 };
287
288 void ShenandoahTraversalGC::flush_liveness(uint worker_id) {
289 jushort* ld = get_liveness(worker_id);
290 for (uint i = 0; i < _heap->num_regions(); i++) {
291 ShenandoahHeapRegion* r = _heap->get_region(i);
292 jushort live = ld[i];
293 if (live > 0) {
294 r->increase_live_data_gc_words(live);
295 ld[i] = 0;
296 }
297 }
298 }
299
300 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
301 _heap(heap),
302 _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())) {
303
304 uint num_queues = heap->max_workers();
305 for (uint i = 0; i < num_queues; ++i) {
306 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
307 task_queue->initialize();
308 _task_queues->register_queue(i, task_queue);
309 }
310
311 uint workers = heap->max_workers();
312 _liveness_local = NEW_C_HEAP_ARRAY(jushort*, workers, mtGC);
313 for (uint worker = 0; worker < workers; worker++) {
314 _liveness_local[worker] = NEW_C_HEAP_ARRAY(jushort, num_regions, mtGC);
315 }
316
317 }
318
319 ShenandoahTraversalGC::~ShenandoahTraversalGC() {
320 }
321
322 void ShenandoahTraversalGC::prepare() {
323 _heap->collection_set()->clear();
324 assert(_heap->collection_set()->count() == 0, "collection set not clear");
325
326 _heap->make_tlabs_parsable(true);
327
328 assert(_heap->is_next_bitmap_clear(), "need clean mark bitmap");
329
330 ShenandoahFreeSet* free_set = _heap->free_set();
331 ShenandoahCollectionSet* collection_set = _heap->collection_set();
332
333 // Find collection set
334 _heap->shenandoahPolicy()->choose_collection_set(collection_set, false);
335
336 // Rebuild free set
337 free_set->rebuild();
338
339 log_info(gc,ergo)("Got "SIZE_FORMAT" collection set regions", collection_set->count());
340 }
341
342 void ShenandoahTraversalGC::init_traversal_collection() {
343 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC");
344
345 if (ShenandoahVerify) {
346 _heap->verifier()->verify_before_traversal();
347 }
348
349 {
350 ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
351 ShenandoahHeapLocker lock(_heap->lock());
352 prepare();
353 }
354
355 _heap->set_concurrent_traversal_in_progress(true);
356
357 bool process_refs = _heap->process_references();
358 if (process_refs) {
359 ReferenceProcessor* rp = _heap->ref_processor();
360 rp->enable_discovery(true /*verify_no_refs*/);
361 rp->setup_policy(false);
362 }
363
364 {
365 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work);
366 assert(_task_queues->is_empty(), "queues must be empty before traversal GC");
367
368 #if defined(COMPILER2) || INCLUDE_JVMCI
369 DerivedPointerTable::clear();
370 #endif
371
372 {
373 uint nworkers = _heap->workers()->active_workers();
374 task_queues()->reserve(nworkers);
375 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
376
377 if (UseShenandoahOWST) {
378 ShenandoahTaskTerminator terminator(nworkers, task_queues());
379 ShenandoahInitTraversalCollectionTask traversal_task(&rp);
380 _heap->workers()->run_task(&traversal_task);
381 } else {
382 ParallelTaskTerminator terminator(nworkers, task_queues());
383 ShenandoahInitTraversalCollectionTask traversal_task(&rp);
384 _heap->workers()->run_task(&traversal_task);
385 }
386 }
387
388 #if defined(COMPILER2) || INCLUDE_JVMCI
389 DerivedPointerTable::update_pointers();
390 #endif
391 }
392
393 if (ShenandoahPacing) {
394 _heap->pacer()->setup_for_traversal();
395 }
396 }
397
398 void ShenandoahTraversalGC::main_loop(uint worker_id, ParallelTaskTerminator* terminator, bool do_satb) {
399 if (do_satb) {
400 main_loop_prework<true>(worker_id, terminator);
401 } else {
402 main_loop_prework<false>(worker_id, terminator);
403 }
404 }
405
406 template <bool DO_SATB>
407 void ShenandoahTraversalGC::main_loop_prework(uint w, ParallelTaskTerminator* t) {
408 ShenandoahObjToScanQueue* q = task_queues()->queue(w);
409
410 // Initialize live data.
411 jushort* ld = get_liveness(w);
412 Copy::fill_to_bytes(ld, _heap->num_regions() * sizeof(jushort));
413
414 ReferenceProcessor* rp = NULL;
415 if (_heap->process_references()) {
416 rp = _heap->ref_processor();
417 }
418 if (!_heap->is_degenerated_gc_in_progress()) {
419 if (_heap->unload_classes()) {
420 if (ShenandoahStringDedup::is_enabled()) {
421 ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
422 ShenandoahTraversalMetadataDedupClosure cl(q, rp, dq);
423 main_loop_work<ShenandoahTraversalMetadataDedupClosure, DO_SATB>(&cl, ld, w, t);
424 } else {
425 ShenandoahTraversalMetadataClosure cl(q, rp);
426 main_loop_work<ShenandoahTraversalMetadataClosure, DO_SATB>(&cl, ld, w, t);
427 }
428 } else {
429 if (ShenandoahStringDedup::is_enabled()) {
430 ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
431 ShenandoahTraversalDedupClosure cl(q, rp, dq);
432 main_loop_work<ShenandoahTraversalDedupClosure, DO_SATB>(&cl, ld, w, t);
433 } else {
434 ShenandoahTraversalClosure cl(q, rp);
435 main_loop_work<ShenandoahTraversalClosure, DO_SATB>(&cl, ld, w, t);
436 }
437 }
438 } else {
439 if (_heap->unload_classes()) {
440 if (ShenandoahStringDedup::is_enabled()) {
441 ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
442 ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp, dq);
443 main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure, DO_SATB>(&cl, ld, w, t);
444 } else {
445 ShenandoahTraversalMetadataDegenClosure cl(q, rp);
446 main_loop_work<ShenandoahTraversalMetadataDegenClosure, DO_SATB>(&cl, ld, w, t);
447 }
448 } else {
449 if (ShenandoahStringDedup::is_enabled()) {
450 ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
451 ShenandoahTraversalDedupDegenClosure cl(q, rp, dq);
452 main_loop_work<ShenandoahTraversalDedupDegenClosure, DO_SATB>(&cl, ld, w, t);
453 } else {
454 ShenandoahTraversalDegenClosure cl(q, rp);
455 main_loop_work<ShenandoahTraversalDegenClosure, DO_SATB>(&cl, ld, w, t);
456 }
457 }
458 }
459 flush_liveness(w);
460
461 }
462
463 template <class T, bool DO_SATB>
464 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ParallelTaskTerminator* terminator) {
465 ShenandoahObjToScanQueueSet* queues = task_queues();
466 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
467 ShenandoahConcurrentMark* conc_mark = _heap->concurrentMark();
468
469 uintx stride = ShenandoahMarkLoopStride;
470
471 ShenandoahMarkTask task;
472
473 // Process outstanding queues, if any.
474 q = queues->claim_next();
475 while (q != NULL) {
476 if (_heap->check_cancelled_concgc_and_yield()) {
477 ShenandoahCancelledTerminatorTerminator tt;
478 ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
479 while (!terminator->offer_termination(&tt));
480 return;
481 }
482
483 for (uint i = 0; i < stride; i++) {
484 if (q->pop_buffer(task) ||
485 q->pop_local(task) ||
486 q->pop_overflow(task)) {
487 conc_mark->do_task<T, true>(q, cl, live_data, &task);
488 } else {
489 assert(q->is_empty(), "Must be empty");
490 q = queues->claim_next();
491 break;
492 }
493 }
494 }
495 // Normal loop.
496 q = queues->queue(worker_id);
497 ShenandoahTraversalSATBBufferClosure satb_cl(q);
498 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
499
500 int seed = 17;
501
502 while (true) {
503 if (check_and_handle_cancelled_gc(terminator)) return;
504
505 for (uint i = 0; i < stride; i++) {
506 if ((q->pop_buffer(task) ||
507 q->pop_local(task) ||
508 q->pop_overflow(task) ||
509 (DO_SATB && satb_mq_set.apply_closure_to_completed_buffer(&satb_cl) && q->pop_buffer(task)) ||
510 queues->steal(worker_id, &seed, task))) {
511 conc_mark->do_task<T, true>(q, cl, live_data, &task);
512 } else {
513 ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
514 if (terminator->offer_termination()) return;
515 }
516 }
517 }
518 }
519
520 bool ShenandoahTraversalGC::check_and_handle_cancelled_gc(ParallelTaskTerminator* terminator) {
521 if (_heap->cancelled_concgc()) {
522 ShenandoahCancelledTerminatorTerminator tt;
523 ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
524 while (! terminator->offer_termination(&tt));
525 return true;
526 }
527 return false;
528 }
529
530 void ShenandoahTraversalGC::concurrent_traversal_collection() {
531 ClassLoaderDataGraph::clear_claimed_marks();
532
533 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal);
534 if (!_heap->cancelled_concgc()) {
535 uint nworkers = _heap->workers()->active_workers();
536 task_queues()->reserve(nworkers);
537 if (UseShenandoahOWST) {
538 ShenandoahTaskTerminator terminator(nworkers, task_queues());
539 ShenandoahConcurrentTraversalCollectionTask traversal_task(&terminator);
540 _heap->workers()->run_task(&traversal_task);
541 } else {
542 ParallelTaskTerminator terminator(nworkers, task_queues());
543 ShenandoahConcurrentTraversalCollectionTask traversal_task(&terminator);
544 _heap->workers()->run_task(&traversal_task);
545 }
546 }
547
548 if (!_heap->cancelled_concgc() && ShenandoahPreclean && _heap->process_references()) {
549 ShenandoahEvacOOMScope oom_evac_scope;
550 preclean_weak_refs();
551 }
552 }
553
554 void ShenandoahTraversalGC::final_traversal_collection() {
555
556 _heap->make_tlabs_parsable(true);
557
558 if (!_heap->cancelled_concgc()) {
559 #if defined(COMPILER2) || INCLUDE_JVMCI
560 DerivedPointerTable::clear();
561 #endif
562 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
563 uint nworkers = _heap->workers()->active_workers();
564 task_queues()->reserve(nworkers);
565
566 // Finish traversal
567 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
568 if (UseShenandoahOWST) {
569 ShenandoahTaskTerminator terminator(nworkers, task_queues());
570 ShenandoahFinalTraversalCollectionTask traversal_task(&rp, &terminator);
571 _heap->workers()->run_task(&traversal_task);
572 } else {
573 ParallelTaskTerminator terminator(nworkers, task_queues());
574 ShenandoahFinalTraversalCollectionTask traversal_task(&rp, &terminator);
575 _heap->workers()->run_task(&traversal_task);
576 }
577 #if defined(COMPILER2) || INCLUDE_JVMCI
578 DerivedPointerTable::update_pointers();
579 #endif
580 }
581
582 if (!_heap->cancelled_concgc() && _heap->process_references()) {
583 weak_refs_work();
584 }
585
586 if (!_heap->cancelled_concgc() && _heap->unload_classes()) {
587 _heap->unload_classes_and_cleanup_tables(false);
588 fixup_roots();
589 }
590
591 if (!_heap->cancelled_concgc()) {
592 // Still good? We can now trash the cset, and make final verification
593 {
594 ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
595 ShenandoahHeapLocker lock(_heap->lock());
596
597 // Trash everything
598 // Clear immediate garbage regions.
599 size_t num_regions = _heap->num_regions();
600
601 ShenandoahFreeSet* free_regions = _heap->free_set();
602 free_regions->clear();
603 for (size_t i = 0; i < num_regions; i++) {
604 ShenandoahHeapRegion* r = _heap->get_region(i);
605 bool not_allocated = _heap->next_top_at_mark_start(r->bottom()) == r->top();
606 if (r->is_humongous_start() && !r->has_live() && not_allocated) {
607 // Trash humongous.
608 HeapWord* humongous_obj = r->bottom() + BrooksPointer::word_size();
609 assert(!_heap->is_marked_next(oop(humongous_obj)), "must not be marked");
610 r->make_trash();
611 while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {
612 i++;
613 r = _heap->get_region(i);
614 assert(r->is_humongous_continuation(), "must be humongous continuation");
615 r->make_trash();
616 }
617 } else if (!r->is_empty() && !r->has_live() && not_allocated) {
618 // Trash regular.
619 assert(!r->is_humongous(), "handled above");
620 assert(!r->is_trash(), "must not already be trashed");
621 r->make_trash();
622 }
623 }
624 _heap->collection_set()->clear();
625 _heap->free_set()->rebuild();
626 reset();
627 }
628
629 if (ShenandoahVerify) {
630 _heap->verifier()->verify_after_traversal();
631 }
632
633 assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
634 _heap->set_concurrent_traversal_in_progress(false);
635 assert(!_heap->cancelled_concgc(), "must not be cancelled when getting out here");
636 }
637 }
638
639 class ShenandoahTraversalFixRootsClosure : public OopClosure {
640 private:
641
642 template <class T>
643 inline void do_oop_work(T* p) {
644 T o = RawAccess<>::oop_load(p);
645 if (!CompressedOops::is_null(o)) {
646 oop obj = CompressedOops::decode_not_null(o);
647 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
648 if (!oopDesc::unsafe_equals(obj, forw)) {
649 RawAccess<OOP_NOT_NULL>::oop_store(p, forw);
650 }
651 }
652 }
653 public:
654 inline void do_oop(oop* p) { do_oop_work(p); }
655 inline void do_oop(narrowOop* p) { do_oop_work(p); }
656 };
657
658 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
659 ShenandoahRootProcessor* _rp;
660 public:
661
662 ShenandoahTraversalFixRootsTask(ShenandoahRootProcessor* rp) :
663 AbstractGangTask("Shenandoah traversal fix roots"),
664 _rp(rp)
665 {
666 // Nothing else to do.
667 }
668
669 void work(uint worker_id) {
670 ShenandoahTraversalFixRootsClosure cl;
671 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
672 CLDToOopClosure cldCl(&cl);
673 _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id);
674 }
675 };
676
677 void ShenandoahTraversalGC::fixup_roots() {
678 #if defined(COMPILER2) || INCLUDE_JVMCI
679 DerivedPointerTable::clear();
680 #endif
681 ShenandoahHeap* heap = ShenandoahHeap::heap();
682 ShenandoahRootProcessor rp(heap, heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots);
683 ShenandoahTraversalFixRootsTask update_roots_task(&rp);
684 heap->workers()->run_task(&update_roots_task);
685 #if defined(COMPILER2) || INCLUDE_JVMCI
686 DerivedPointerTable::update_pointers();
687 #endif
688 }
689
690 void ShenandoahTraversalGC::reset() {
691 _task_queues->clear();
692 }
693
694 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
695 return _task_queues;
696 }
697
698 jushort* ShenandoahTraversalGC::get_liveness(uint worker_id) {
699 return _liveness_local[worker_id];
700 }
701
702 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
703 private:
704 ShenandoahHeap* const _heap;
705 public:
706 ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
707 virtual bool should_return() { return _heap->cancelled_concgc(); }
708 };
709
710 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
711 public:
712 void do_void() {
713 ShenandoahHeap* sh = ShenandoahHeap::heap();
714 ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
715 assert(sh->process_references(), "why else would we be here?");
716 ParallelTaskTerminator terminator(1, traversal_gc->task_queues());
717 shenandoah_assert_rp_isalive_installed();
718 traversal_gc->main_loop((uint) 0, &terminator, false);
719 }
720 };
721
722 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
723 private:
724 ShenandoahObjToScanQueue* _queue;
725 Thread* _thread;
726 ShenandoahTraversalGC* _traversal_gc;
727 template <class T>
728 inline void do_oop_nv(T* p) {
729 _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue);
730 }
731
732 public:
733 ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
734 _queue(q), _thread(Thread::current()),
735 _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {}
736
737 void do_oop(narrowOop* p) { do_oop_nv(p); }
738 void do_oop(oop* p) { do_oop_nv(p); }
739 };
740
741 class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure {
742 private:
743 ShenandoahObjToScanQueue* _queue;
744 Thread* _thread;
745 ShenandoahTraversalGC* _traversal_gc;
746 template <class T>
747 inline void do_oop_nv(T* p) {
748 _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue);
749 }
750
751 public:
752 ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
753 _queue(q), _thread(Thread::current()),
754 _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {}
755
756 void do_oop(narrowOop* p) { do_oop_nv(p); }
757 void do_oop(oop* p) { do_oop_nv(p); }
758 };
759
760 void ShenandoahTraversalGC::preclean_weak_refs() {
761 // Pre-cleaning weak references before diving into STW makes sense at the
762 // end of concurrent mark. This will filter out the references which referents
763 // are alive. Note that ReferenceProcessor already filters out these on reference
764 // discovery, and the bulk of work is done here. This phase processes leftovers
765 // that missed the initial filtering, i.e. when referent was marked alive after
766 // reference was discovered by RP.
767
768 assert(_heap->process_references(), "sanity");
769
770 ShenandoahHeap* sh = ShenandoahHeap::heap();
771 ReferenceProcessor* rp = sh->ref_processor();
772
773 // Shortcut if no references were discovered to avoid winding up threads.
774 if (!rp->has_discovered_references()) {
775 return;
776 }
777
778 ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
779
780 shenandoah_assert_rp_isalive_not_installed();
781 ReferenceProcessorIsAliveMutator fix_isalive(rp, sh->is_alive_closure());
782
783 // Interrupt on cancelled GC
784 ShenandoahTraversalCancelledGCYieldClosure yield;
785
786 assert(task_queues()->is_empty(), "Should be empty");
787 assert(!sh->is_degenerated_gc_in_progress(), "must be in concurrent non-degenerated phase");
788
789 ShenandoahTraversalPrecleanCompleteGCClosure complete_gc;
790 ShenandoahForwardedIsAliveClosure is_alive;
791 ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(0));
792 ResourceMark rm;
793 rp->preclean_discovered_references(&is_alive, &keep_alive,
794 &complete_gc, &yield,
795 NULL);
796 assert(!sh->cancelled_concgc() || task_queues()->is_empty(), "Should be empty");
797 }
798
799 // Weak Reference Closures
800 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
801 uint _worker_id;
802 ParallelTaskTerminator* _terminator;
803 bool _reset_terminator;
804
805 public:
806 ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ParallelTaskTerminator* t, bool reset_terminator = false):
807 _worker_id(worker_id),
808 _terminator(t),
809 _reset_terminator(reset_terminator) {
810 }
811
812 void do_void() {
813 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
814
815 ShenandoahHeap* sh = ShenandoahHeap::heap();
816 ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
817 assert(sh->process_references(), "why else would we be here?");
818 shenandoah_assert_rp_isalive_installed();
819
820 traversal_gc->main_loop(_worker_id, _terminator, false);
821
822 if (_reset_terminator) {
823 _terminator->reset_for_reuse();
824 }
825 }
826 };
827
828 void ShenandoahTraversalGC::weak_refs_work() {
829 assert(_heap->process_references(), "sanity");
830
831 ShenandoahHeap* sh = ShenandoahHeap::heap();
832
833 ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
834
835 ShenandoahGCPhase phase(phase_root);
836
837 ReferenceProcessor* rp = sh->ref_processor();
838
839 // NOTE: We cannot shortcut on has_discovered_references() here, because
840 // we will miss marking JNI Weak refs then, see implementation in
841 // ReferenceProcessor::process_discovered_references.
842 weak_refs_work_doit();
843
844 rp->verify_no_references_recorded();
845 assert(!rp->discovery_enabled(), "Post condition");
846
847 }
848
849 class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask {
850
851 private:
852 AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
853 ParallelTaskTerminator* _terminator;
854 public:
855
856 ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
857 ParallelTaskTerminator* t) :
858 AbstractGangTask("Process reference objects in parallel"),
859 _proc_task(proc_task),
860 _terminator(t) {
861 }
862
863 void work(uint worker_id) {
864 ShenandoahEvacOOMScope oom_evac_scope;
865 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
866 ShenandoahHeap* heap = ShenandoahHeap::heap();
867 ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator);
868
869 ShenandoahForwardedIsAliveClosure is_alive;
870 if (!heap->is_degenerated_gc_in_progress()) {
871 ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
872 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
873 } else {
874 ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
875 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
876 }
877 }
878 };
879
880 class ShenandoahTraversalRefEnqueueTaskProxy : public AbstractGangTask {
881
882 private:
883 AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task;
884
885 public:
886
887 ShenandoahTraversalRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) :
888 AbstractGangTask("Enqueue reference objects in parallel"),
889 _enqueue_task(enqueue_task) {
890 }
891
892 void work(uint worker_id) {
893 _enqueue_task.work(worker_id);
894 }
895 };
896
897 class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
898
899 private:
900 WorkGang* _workers;
901
902 public:
903
904 ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) :
905 _workers(workers) {
906 }
907
908 // Executes a task using worker threads.
909 void execute(ProcessTask& task) {
910 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
911
912 // Shortcut execution if task is empty.
913 // This should be replaced with the generic ReferenceProcessor shortcut,
914 // see JDK-8181214, JDK-8043575, JDK-6938732.
915 if (task.is_empty()) {
916 return;
917 }
918
919 ShenandoahHeap* heap = ShenandoahHeap::heap();
920 ShenandoahTraversalGC* traversal_gc = heap->traversal_gc();
921 uint nworkers = _workers->active_workers();
922 traversal_gc->task_queues()->reserve(nworkers);
923 if (UseShenandoahOWST) {
924 ShenandoahTaskTerminator terminator(nworkers, traversal_gc->task_queues());
925 ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
926 _workers->run_task(&proc_task_proxy);
927 } else {
928 ParallelTaskTerminator terminator(nworkers, traversal_gc->task_queues());
929 ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
930 _workers->run_task(&proc_task_proxy);
931 }
932 }
933
934 void execute(EnqueueTask& task) {
935 ShenandoahTraversalRefEnqueueTaskProxy enqueue_task_proxy(task);
936 _workers->run_task(&enqueue_task_proxy);
937 }
938 };
939
940 void ShenandoahTraversalGC::weak_refs_work_doit() {
941 ShenandoahHeap* sh = ShenandoahHeap::heap();
942
943 ReferenceProcessor* rp = sh->ref_processor();
944
945 ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process;
946 ShenandoahPhaseTimings::Phase phase_enqueue = ShenandoahPhaseTimings::weakrefs_enqueue;
947
948 shenandoah_assert_rp_isalive_not_installed();
949 ReferenceProcessorIsAliveMutator fix_isalive(rp, sh->is_alive_closure());
950
951 WorkGang* workers = sh->workers();
952 uint nworkers = workers->active_workers();
953
954 // Setup collector policy for softref cleaning.
955 bool clear_soft_refs = sh->soft_ref_policy()->use_should_clear_all_soft_refs(true /* bogus arg*/);
956 log_develop_debug(gc, ref)("clearing soft refs: %s", BOOL_TO_STR(clear_soft_refs));
957 rp->setup_policy(clear_soft_refs);
958 rp->set_active_mt_degree(nworkers);
959
960 assert(task_queues()->is_empty(), "Should be empty");
961
962 // complete_gc and keep_alive closures instantiated here are only needed for
963 // single-threaded path in RP. They share the queue 0 for tracking work, which
964 // simplifies implementation. Since RP may decide to call complete_gc several
965 // times, we need to be able to reuse the terminator.
966 uint serial_worker_id = 0;
967 ParallelTaskTerminator terminator(1, task_queues());
968 ShenandoahTraversalDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
969
970 ShenandoahTraversalRefProcTaskExecutor executor(workers);
971
972 ReferenceProcessorPhaseTimes pt(sh->gc_timer(), rp->num_q());
973
974 {
975 ShenandoahGCPhase phase(phase_process);
976
977 ShenandoahForwardedIsAliveClosure is_alive;
978 ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id));
979 rp->process_discovered_references(&is_alive, &keep_alive,
980 &complete_gc, &executor,
981 &pt);
982 pt.print_all_references();
983
984 WeakProcessor::weak_oops_do(&is_alive, &keep_alive);
985
986 assert(!_heap->cancelled_concgc() || task_queues()->is_empty(), "Should be empty");
987 }
988
989 if (_heap->cancelled_concgc()) return;
990
991 {
992 ShenandoahGCPhase phase(phase_enqueue);
993 rp->enqueue_discovered_references(&executor, &pt);
994 pt.print_enqueue_phase();
995 }
996 }