1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shared/gcTraceTime.inline.hpp"
27 #include "gc/shared/markBitMap.inline.hpp"
28 #include "gc/shared/workgroup.hpp"
29 #include "gc/shared/taskqueue.inline.hpp"
30 #include "gc/shared/weakProcessor.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahFreeSet.hpp"
35 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
37 #include "gc/shenandoah/shenandoahHeap.hpp"
38 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
40 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
41 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
42 #include "gc/shenandoah/shenandoahStringDedup.hpp"
43 #include "gc/shenandoah/shenandoahStrDedupQueue.inline.hpp"
44 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
45 #include "gc/shenandoah/shenandoahUtils.hpp"
46 #include "gc/shenandoah/shenandoahVerifier.hpp"
47 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
48 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
49
50 #include "memory/iterator.hpp"
51
52 /**
53 * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
54 * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
55 * is incremental-update-based.
56 *
57 * NOTE on interaction with TAMS: we want to avoid traversing new objects for
58 * several reasons:
59 * - We will not reclaim them in this cycle anyway, because they are not in the
60 * cset
61 * - It makes up for the bulk of work during final-pause
62 * - It also shortens the concurrent cycle because we don't need to
63 * pointlessly traverse through newly allocated objects.
64 * - As a nice side-effect, it solves the I-U termination problem (mutators
65 * cannot outrun the GC by allocating like crazy)
66 * - It is an easy way to achieve MWF. What MWF does is to also enqueue the
67 * target object of stores if it's new. Treating new objects live implicitely
68 * achieves the same, but without extra barriers. I think the effect of
69 * shortened final-pause (mentioned above) is the main advantage of MWF. In
70 * particular, we will not see the head of a completely new long linked list
71 * in final-pause and end up traversing huge chunks of the heap there.
72 * - We don't need to see/update the fields of new objects either, because they
73 * are either still null, or anything that's been stored into them has been
74 * evacuated+enqueued before (and will thus be treated later).
75 *
76 * We achieve this by setting TAMS for each region, and everything allocated
77 * beyond TAMS will be 'implicitely marked'.
78 *
79 * Gotchas:
80 * - While we want new objects to be implicitely marked, we don't want to count
81 * them alive. Otherwise the next cycle wouldn't pick them up and consider
82 * them for cset. This means that we need to protect such regions from
83 * getting accidentally thrashed at the end of traversal cycle. This is why I
84 * keep track of alloc-regions and check is_alloc_region() in the trashing
85 * code.
86 * - We *need* to traverse through evacuated objects. Those objects are
87 * pre-existing, and any references in them point to interesting objects that
88 * we need to see. We also want to count them as live, because we just
89 * determined that they are alive :-) I achieve this by upping TAMS
90 * concurrently for every gclab/gc-shared alloc before publishing the
91 * evacuated object. This way, the GC threads will not consider such objects
92 * implictely marked, and traverse through them as normal.
93 */
94 class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure {
95 private:
96 ShenandoahObjToScanQueue* _queue;
97 ShenandoahTraversalGC* _traversal_gc;
98 ShenandoahHeap* _heap;
99 public:
100 ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) :
101 _queue(q), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
102 _heap(ShenandoahHeap::heap())
103 { }
104
105 void do_buffer(void** buffer, size_t size) {
106 for (size_t i = 0; i < size; ++i) {
107 oop* p = (oop*) &buffer[i];
108 oop obj = oopDesc::load_heap_oop(p);
109 shenandoah_assert_not_forwarded(p, obj);
110 if (!_heap->is_marked_next(obj) && _heap->mark_next(obj)) {
111 _queue->push(ShenandoahMarkTask(obj));
112 }
113 }
114 }
115 };
116
117 class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure {
118 ShenandoahTraversalSATBBufferClosure* _satb_cl;
119
120 public:
121 ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) :
122 _satb_cl(satb_cl) {}
123
124 void do_thread(Thread* thread) {
125 if (thread->is_Java_thread()) {
126 JavaThread* jt = (JavaThread*)thread;
127 jt->satb_mark_queue().apply_closure_and_empty(_satb_cl);
128 } else if (thread->is_VM_thread()) {
129 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
130 }
131 }
132 };
133
134 // Like CLDToOopClosure, but clears has_modified_oops, so that we can record modified CLDs during traversal
135 // and remark them later during final-traversal.
136 class ShenandoahMarkCLDClosure : public CLDClosure {
137 private:
138 OopClosure* _cl;
139 public:
140 ShenandoahMarkCLDClosure(OopClosure* cl) : _cl(cl) {}
141 void do_cld(ClassLoaderData* cld) {
142 cld->oops_do(_cl, true, true);
143 }
144 };
145
146 // Like CLDToOopClosure, but only process modified CLDs
147 class ShenandoahRemarkCLDClosure : public CLDClosure {
148 private:
149 OopClosure* _cl;
150 public:
151 ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {}
152 void do_cld(ClassLoaderData* cld) {
153 if (cld->has_modified_oops()) {
154 cld->oops_do(_cl, true, true);
155 }
156 }
157 };
158
159 class ShenandoahInitTraversalCollectionTask : public AbstractGangTask {
160 private:
161 ShenandoahRootProcessor* _rp;
162 ShenandoahHeap* _heap;
163 public:
164 ShenandoahInitTraversalCollectionTask(ShenandoahRootProcessor* rp) :
165 AbstractGangTask("Shenandoah Init Traversal Collection"),
166 _rp(rp),
167 _heap(ShenandoahHeap::heap()) {}
168
169 void work(uint worker_id) {
170 ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
171 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
172
173 // Initialize live data.
174 jushort* ld = _heap->traversal_gc()->get_liveness(worker_id);
175 Copy::fill_to_bytes(ld, _heap->num_regions() * sizeof(jushort));
176
177 bool process_refs = _heap->shenandoahPolicy()->process_references();
178 bool unload_classes = _heap->shenandoahPolicy()->unload_classes();
179 ReferenceProcessor* rp = NULL;
180 if (process_refs) {
181 rp = _heap->ref_processor();
182 }
183
184 // Step 1: Process ordinary GC roots.
185 {
186 ShenandoahTraversalClosure roots_cl(q, rp);
187 ShenandoahMarkCLDClosure cld_cl(&roots_cl);
188 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
189 if (unload_classes) {
190 _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, &code_cl, NULL, worker_id);
191 } else {
192 _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, NULL, worker_id);
193 }
194 }
195 }
196 };
197
198 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
199 private:
200 ParallelTaskTerminator* _terminator;
201 ShenandoahHeap* _heap;
202 public:
203 ShenandoahConcurrentTraversalCollectionTask(ParallelTaskTerminator* terminator) :
204 AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
205 _terminator(terminator),
206 _heap(ShenandoahHeap::heap()) {}
207
208 void work(uint worker_id) {
209 ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
210 ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
211 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
212
213 // Drain all outstanding work in queues.
214 traversal_gc->main_loop(worker_id, _terminator, true);
215 }
216 };
217
218 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
219 private:
220 ShenandoahRootProcessor* _rp;
221 ParallelTaskTerminator* _terminator;
222 ShenandoahHeap* _heap;
223 public:
224 ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ParallelTaskTerminator* terminator) :
225 AbstractGangTask("Shenandoah Final Traversal Collection"),
226 _rp(rp),
227 _terminator(terminator),
228 _heap(ShenandoahHeap::heap()) {}
229
230 void work(uint worker_id) {
231 ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
232
233 ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
234 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
235
236 bool process_refs = _heap->shenandoahPolicy()->process_references();
237 bool unload_classes = _heap->shenandoahPolicy()->unload_classes();
238 ReferenceProcessor* rp = NULL;
239 if (process_refs) {
240 rp = _heap->ref_processor();
241 }
242
243 // Step 1: Drain outstanding SATB queues.
244 // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
245 ShenandoahTraversalSATBBufferClosure satb_cl(q);
246 {
247 // Process remaining finished SATB buffers.
248 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
249 while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
250 // Process remaining threads SATB buffers below.
251 }
252
253 // Step 1: Process ordinary GC roots.
254 {
255 ShenandoahTraversalClosure roots_cl(q, rp);
256 CLDToOopClosure cld_cl(&roots_cl);
257 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
258 ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
259 if (unload_classes) {
260 ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
261 _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, &code_cl, &tc, worker_id);
262 } else {
263 _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, &tc, worker_id);
264 }
265 }
266
267 {
268 ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
269 ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
270
271 // Step 3: Finally drain all outstanding work in queues.
272 traversal_gc->main_loop(worker_id, _terminator, false);
273 }
274
275 // Flush remaining liveness data.
276 traversal_gc->flush_liveness(worker_id);
277
278 }
279 };
280
281 void ShenandoahTraversalGC::flush_liveness(uint worker_id) {
282 jushort* ld = get_liveness(worker_id);
283 for (uint i = 0; i < _heap->regions()->active_regions(); i++) {
284 ShenandoahHeapRegion* r = _heap->regions()->get(i);
285 jushort live = ld[i];
286 if (live > 0) {
287 r->increase_live_data_words(live);
288 ld[i] = 0;
289 }
290 }
291 }
292
293 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
294 _heap(heap),
295 _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())) {
296
297 uint num_queues = heap->max_workers();
298 for (uint i = 0; i < num_queues; ++i) {
299 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
300 task_queue->initialize();
301 _task_queues->register_queue(i, task_queue);
302 }
303
304 uint workers = heap->max_workers();
305 _liveness_local = NEW_C_HEAP_ARRAY(jushort*, workers, mtGC);
306 for (uint worker = 0; worker < workers; worker++) {
307 _liveness_local[worker] = NEW_C_HEAP_ARRAY(jushort, num_regions, mtGC);
308 }
309
310 }
311
312 ShenandoahTraversalGC::~ShenandoahTraversalGC() {
313 }
314
315 void ShenandoahTraversalGC::prepare() {
316 _heap->collection_set()->clear();
317 assert(_heap->collection_set()->count() == 0, "collection set not clear");
318
319 _heap->make_tlabs_parsable(true);
320
321 assert(_heap->is_next_bitmap_clear(), "need clean mark bitmap");
322
323 ShenandoahHeapRegionSet* regions = _heap->regions();
324 ShenandoahCollectionSet* collection_set = _heap->collection_set();
325 size_t num_regions = _heap->num_regions();
326
327 // Find collection set
328 _heap->shenandoahPolicy()->choose_collection_set(collection_set, false);
329
330 // Rebuild free set
331 ShenandoahFreeSet* _free_regions = _heap->free_regions();
332 _free_regions->clear();
333
334 for (uint from_idx = 0; from_idx < num_regions; from_idx++) {
335 ShenandoahHeapRegion* r = regions->get(from_idx);
336 if (r->is_alloc_allowed()) {
337 _free_regions->add_region(r);
338 }
339 }
340
341 log_info(gc,ergo)("Got "SIZE_FORMAT" collection set regions", collection_set->count());
342 }
343
344 void ShenandoahTraversalGC::init_traversal_collection() {
345 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC");
346
347 if (ShenandoahVerify) {
348 _heap->verifier()->verify_before_traversal();
349 }
350
351 {
352 ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
353 ShenandoahHeapLocker lock(_heap->lock());
354 prepare();
355 }
356
357 _heap->set_concurrent_traversal_in_progress(true);
358
359 bool process_refs = _heap->shenandoahPolicy()->process_references();
360 if (process_refs) {
361 ReferenceProcessor* rp = _heap->ref_processor();
362 rp->enable_discovery(true /*verify_no_refs*/);
363 rp->setup_policy(false);
364 }
365
366 {
367 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work);
368 assert(_task_queues->is_empty(), "queues must be empty before traversal GC");
369
370 #if defined(COMPILER2) || INCLUDE_JVMCI
371 DerivedPointerTable::clear();
372 #endif
373
374 {
375 uint nworkers = _heap->workers()->active_workers();
376 task_queues()->reserve(nworkers);
377 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
378
379 if (UseShenandoahOWST) {
380 ShenandoahTaskTerminator terminator(nworkers, task_queues());
381 ShenandoahInitTraversalCollectionTask traversal_task(&rp);
382 _heap->workers()->run_task(&traversal_task);
383 } else {
384 ParallelTaskTerminator terminator(nworkers, task_queues());
385 ShenandoahInitTraversalCollectionTask traversal_task(&rp);
386 _heap->workers()->run_task(&traversal_task);
387 }
388 }
389
390 #if defined(COMPILER2) || INCLUDE_JVMCI
391 DerivedPointerTable::update_pointers();
392 #endif
393 if (_heap->cancelled_concgc()) {
394 _heap->fixup_roots();
395 reset();
396 _heap->set_concurrent_traversal_in_progress(false);
397 }
398 }
399 }
400
401 void ShenandoahTraversalGC::main_loop(uint worker_id, ParallelTaskTerminator* terminator, bool do_satb) {
402 if (do_satb) {
403 main_loop_prework<true>(worker_id, terminator);
404 } else {
405 main_loop_prework<false>(worker_id, terminator);
406 }
407 }
408
409 template <bool DO_SATB>
410 void ShenandoahTraversalGC::main_loop_prework(uint w, ParallelTaskTerminator* t) {
411 ShenandoahObjToScanQueue* q = task_queues()->queue(w);
412 jushort* ld = get_liveness(w);
413
414 ReferenceProcessor* rp = NULL;
415 if (_heap->shenandoahPolicy()->process_references()) {
416 rp = _heap->ref_processor();
417 }
418 if (_heap->shenandoahPolicy()->unload_classes()) {
419 if (ShenandoahStringDedup::is_enabled()) {
420 ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
421 ShenandoahTraversalMetadataDedupClosure cl(q, rp, dq);
422 main_loop_work<ShenandoahTraversalMetadataDedupClosure, DO_SATB>(&cl, ld, w, t);
423 } else {
424 ShenandoahTraversalMetadataClosure cl(q, rp);
425 main_loop_work<ShenandoahTraversalMetadataClosure, DO_SATB>(&cl, ld, w, t);
426 }
427 } else {
428 if (ShenandoahStringDedup::is_enabled()) {
429 ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
430 ShenandoahTraversalDedupClosure cl(q, rp, dq);
431 main_loop_work<ShenandoahTraversalDedupClosure, DO_SATB>(&cl, ld, w, t);
432 } else {
433 ShenandoahTraversalClosure cl(q, rp);
434 main_loop_work<ShenandoahTraversalClosure, DO_SATB>(&cl, ld, w, t);
435 }
436 }
437 }
438
439 template <class T, bool DO_SATB>
440 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ParallelTaskTerminator* terminator) {
441 ShenandoahObjToScanQueueSet* queues = task_queues();
442 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
443
444 uintx stride = ShenandoahMarkLoopStride;
445
446 ShenandoahMarkTask task;
447
448 // Process outstanding queues, if any.
449 q = queues->claim_next();
450 while (q != NULL) {
451 if (_heap->check_cancelled_concgc_and_yield()) {
452 ShenandoahCancelledTerminatorTerminator tt;
453 while (!terminator->offer_termination(&tt));
454 return;
455 }
456
457 for (uint i = 0; i < stride; i++) {
458 if (q->pop_buffer(task) ||
459 q->pop_local(task) ||
460 q->pop_overflow(task)) {
461 do_task(q, cl, live_data, &task);
462 } else {
463 assert(q->is_empty(), "Must be empty");
464 q = queues->claim_next();
465 break;
466 }
467 }
468 }
469 // Normal loop.
470 q = queues->queue(worker_id);
471 ShenandoahTraversalSATBBufferClosure satb_cl(q);
472 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
473
474 int seed = 17;
475
476 while (true) {
477 if (check_and_handle_cancelled_gc(terminator)) return;
478
479 for (uint i = 0; i < stride; i++) {
480 if ((q->pop_buffer(task) ||
481 q->pop_local(task) ||
482 q->pop_overflow(task) ||
483 (DO_SATB && satb_mq_set.apply_closure_to_completed_buffer(&satb_cl) && q->pop_buffer(task)) ||
484 queues->steal(worker_id, &seed, task))) {
485 do_task(q, cl, live_data, &task);
486 } else {
487 if (terminator->offer_termination()) return;
488 }
489 }
490 }
491 }
492
493 bool ShenandoahTraversalGC::check_and_handle_cancelled_gc(ParallelTaskTerminator* terminator) {
494 if (_heap->cancelled_concgc()) {
495 ShenandoahCancelledTerminatorTerminator tt;
496 while (! terminator->offer_termination(&tt));
497 return true;
498 }
499 return false;
500 }
501
502 void ShenandoahTraversalGC::concurrent_traversal_collection() {
503 ClassLoaderDataGraph::clear_claimed_marks();
504
505 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal);
506 if (!_heap->cancelled_concgc()) {
507 uint nworkers = _heap->workers()->active_workers();
508 task_queues()->reserve(nworkers);
509 if (UseShenandoahOWST) {
510 ShenandoahTaskTerminator terminator(nworkers, task_queues());
511 ShenandoahConcurrentTraversalCollectionTask traversal_task(&terminator);
512 _heap->workers()->run_task(&traversal_task);
513 } else {
514 ParallelTaskTerminator terminator(nworkers, task_queues());
515 ShenandoahConcurrentTraversalCollectionTask traversal_task(&terminator);
516 _heap->workers()->run_task(&traversal_task);
517 }
518 }
519
520 if (!_heap->cancelled_concgc() && ShenandoahPreclean && _heap->shenandoahPolicy()->process_references()) {
521 preclean_weak_refs();
522 }
523
524 if (_heap->cancelled_concgc()) {
525 _task_queues->clear();
526 }
527 assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
528 }
529
530 void ShenandoahTraversalGC::final_traversal_collection() {
531
532 _heap->make_tlabs_parsable(true);
533
534 if (!_heap->cancelled_concgc()) {
535 #if defined(COMPILER2) || INCLUDE_JVMCI
536 DerivedPointerTable::clear();
537 #endif
538 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
539 uint nworkers = _heap->workers()->active_workers();
540 task_queues()->reserve(nworkers);
541
542 // Finish traversal
543 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
544 if (UseShenandoahOWST) {
545 ShenandoahTaskTerminator terminator(nworkers, task_queues());
546 ShenandoahFinalTraversalCollectionTask traversal_task(&rp, &terminator);
547 _heap->workers()->run_task(&traversal_task);
548 } else {
549 ParallelTaskTerminator terminator(nworkers, task_queues());
550 ShenandoahFinalTraversalCollectionTask traversal_task(&rp, &terminator);
551 _heap->workers()->run_task(&traversal_task);
552 }
553 #if defined(COMPILER2) || INCLUDE_JVMCI
554 DerivedPointerTable::update_pointers();
555 #endif
556 }
557
558 if (!_heap->cancelled_concgc() && _heap->shenandoahPolicy()->process_references()) {
559 weak_refs_work();
560 }
561
562 if (!_heap->cancelled_concgc() && _heap->shenandoahPolicy()->unload_classes()) {
563 _heap->unload_classes_and_cleanup_tables(false);
564 _heap->concurrentMark()->update_roots(ShenandoahPhaseTimings::final_traversal_update_roots);
565 }
566
567 if (!_heap->cancelled_concgc()) {
568 // Still good? We can now trash the cset, and make final verification
569 {
570 ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
571 ShenandoahHeapLocker lock(_heap->lock());
572
573 // Trash everything
574 // Clear immediate garbage regions.
575 ShenandoahHeapRegionSet* regions = _heap->regions();
576 size_t active = regions->active_regions();
577 ShenandoahFreeSet* free_regions = _heap->free_regions();
578 free_regions->clear();
579 for (size_t i = 0; i < active; i++) {
580 ShenandoahHeapRegion* r = regions->get(i);
581 bool not_allocated = _heap->next_top_at_mark_start(r->bottom()) == r->top();
582 if (r->is_humongous_start() && !r->has_live() && not_allocated) {
583 // Trash humongous.
584 HeapWord* humongous_obj = r->bottom() + BrooksPointer::word_size();
585 assert(!_heap->is_marked_next(oop(humongous_obj)), "must not be marked");
586 r->make_trash();
587 while (i + 1 < active && regions->get(i + 1)->is_humongous_continuation()) {
588 i++;
589 r = regions->get(i);
590 assert(r->is_humongous_continuation(), "must be humongous continuation");
591 r->make_trash();
592 }
593 } else if (!r->is_empty() && !r->has_live() && not_allocated) {
594 // Trash regular.
595 assert(!r->is_humongous(), "handled above");
596 assert(!r->is_trash(), "must not already be trashed");
597 r->make_trash();
598 } else if (r->is_alloc_allowed()) {
599 free_regions->add_region(r);
600 }
601 }
602 _heap->collection_set()->clear();
603 reset();
604 }
605
606 if (ShenandoahVerify) {
607 _heap->verifier()->verify_after_traversal();
608 }
609 } else {
610 // On cancellation path, fixup roots to make them consistent
611 _heap->fixup_roots();
612 reset();
613 }
614
615 assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
616 _heap->set_concurrent_traversal_in_progress(false);
617 }
618
619 void ShenandoahTraversalGC::reset() {
620 _task_queues->clear();
621 }
622
623 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
624 return _task_queues;
625 }
626
627 jushort* ShenandoahTraversalGC::get_liveness(uint worker_id) {
628 return _liveness_local[worker_id];
629 }
630
631 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
632 private:
633 ShenandoahHeap* const _heap;
634 public:
635 ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
636 virtual bool should_return() { return _heap->cancelled_concgc(); }
637 };
638
639 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
640 public:
641 void do_void() {
642 ShenandoahHeap* sh = ShenandoahHeap::heap();
643 ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
644 assert(sh->shenandoahPolicy()->process_references(), "why else would we be here?");
645 ReferenceProcessor* rp = sh->ref_processor();
646 ParallelTaskTerminator terminator(1, traversal_gc->task_queues());
647 ReferenceProcessorIsAliveMutator fix_alive(rp, sh->is_alive_closure());
648 traversal_gc->main_loop((uint) 0, &terminator, false);
649 }
650 };
651
652 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
653 private:
654 ShenandoahObjToScanQueue* _queue;
655 Thread* _thread;
656 ShenandoahTraversalGC* _traversal_gc;
657 template <class T>
658 inline void do_oop_nv(T* p) {
659 _traversal_gc->process_oop<T, false /* string dedup */>(p, _thread, _queue);
660 }
661
662 public:
663 ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
664 _queue(q), _thread(Thread::current()),
665 _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {}
666
667 void do_oop(narrowOop* p) { do_oop_nv(p); }
668 void do_oop(oop* p) { do_oop_nv(p); }
669 };
670
671 void ShenandoahTraversalGC::preclean_weak_refs() {
672 // Pre-cleaning weak references before diving into STW makes sense at the
673 // end of concurrent mark. This will filter out the references which referents
674 // are alive. Note that ReferenceProcessor already filters out these on reference
675 // discovery, and the bulk of work is done here. This phase processes leftovers
676 // that missed the initial filtering, i.e. when referent was marked alive after
677 // reference was discovered by RP.
678
679 assert(_heap->shenandoahPolicy()->process_references(), "sanity");
680
681 ShenandoahHeap* sh = ShenandoahHeap::heap();
682 ReferenceProcessor* rp = sh->ref_processor();
683
684 // Shortcut if no references were discovered to avoid winding up threads.
685 if (!rp->has_discovered_references()) {
686 return;
687 }
688
689 ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
690 ReferenceProcessorIsAliveMutator fix_alive(rp, sh->is_alive_closure());
691
692 // Interrupt on cancelled GC
693 ShenandoahTraversalCancelledGCYieldClosure yield;
694
695 assert(task_queues()->is_empty(), "Should be empty");
696
697 ShenandoahTraversalPrecleanCompleteGCClosure complete_gc;
698 ShenandoahForwardedIsAliveClosure is_alive;
699 ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(0));
700 ResourceMark rm;
701 rp->preclean_discovered_references(&is_alive, &keep_alive,
702 &complete_gc, &yield,
703 NULL);
704 assert(!sh->cancelled_concgc() || task_queues()->is_empty(), "Should be empty");
705 }
706
707 // Weak Reference Closures
708 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
709 uint _worker_id;
710 ParallelTaskTerminator* _terminator;
711 bool _reset_terminator;
712
713 public:
714 ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ParallelTaskTerminator* t, bool reset_terminator = false):
715 _worker_id(worker_id),
716 _terminator(t),
717 _reset_terminator(reset_terminator) {
718 }
719
720 void do_void() {
721 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
722
723 ShenandoahHeap* sh = ShenandoahHeap::heap();
724 ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
725 assert(sh->shenandoahPolicy()->process_references(), "why else would we be here?");
726 ReferenceProcessor* rp = sh->ref_processor();
727 ReferenceProcessorIsAliveMutator fix_alive(rp, sh->is_alive_closure());
728
729 traversal_gc->main_loop(_worker_id, _terminator, false);
730 traversal_gc->flush_liveness(_worker_id);
731
732 if (_reset_terminator) {
733 _terminator->reset_for_reuse();
734 }
735 }
736 };
737
738 void ShenandoahTraversalGC::weak_refs_work() {
739 assert(_heap->shenandoahPolicy()->process_references(), "sanity");
740
741 ShenandoahHeap* sh = ShenandoahHeap::heap();
742
743 ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
744
745 ShenandoahGCPhase phase(phase_root);
746
747 ReferenceProcessor* rp = sh->ref_processor();
748
749 // NOTE: We cannot shortcut on has_discovered_references() here, because
750 // we will miss marking JNI Weak refs then, see implementation in
751 // ReferenceProcessor::process_discovered_references.
752 weak_refs_work_doit();
753
754 rp->verify_no_references_recorded();
755 assert(!rp->discovery_enabled(), "Post condition");
756
757 }
758
759 class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask {
760
761 private:
762 AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
763 ParallelTaskTerminator* _terminator;
764 public:
765
766 ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
767 ParallelTaskTerminator* t) :
768 AbstractGangTask("Process reference objects in parallel"),
769 _proc_task(proc_task),
770 _terminator(t) {
771 }
772
773 void work(uint worker_id) {
774 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
775 ShenandoahHeap* heap = ShenandoahHeap::heap();
776 ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator);
777
778 ShenandoahForwardedIsAliveClosure is_alive;
779 ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
780 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
781 }
782 };
783
784 class ShenandoahTraversalRefEnqueueTaskProxy : public AbstractGangTask {
785
786 private:
787 AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task;
788
789 public:
790
791 ShenandoahTraversalRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) :
792 AbstractGangTask("Enqueue reference objects in parallel"),
793 _enqueue_task(enqueue_task) {
794 }
795
796 void work(uint worker_id) {
797 _enqueue_task.work(worker_id);
798 }
799 };
800
801 class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
802
803 private:
804 WorkGang* _workers;
805
806 public:
807
808 ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) :
809 _workers(workers) {
810 }
811
812 // Executes a task using worker threads.
813 void execute(ProcessTask& task) {
814 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
815
816 // Shortcut execution if task is empty.
817 // This should be replaced with the generic ReferenceProcessor shortcut,
818 // see JDK-8181214, JDK-8043575, JDK-6938732.
819 if (task.is_empty()) {
820 return;
821 }
822
823 ShenandoahHeap* heap = ShenandoahHeap::heap();
824 ShenandoahTraversalGC* traversal_gc = heap->traversal_gc();
825 uint nworkers = _workers->active_workers();
826 traversal_gc->task_queues()->reserve(nworkers);
827 if (UseShenandoahOWST) {
828 ShenandoahTaskTerminator terminator(nworkers, traversal_gc->task_queues());
829 ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
830 _workers->run_task(&proc_task_proxy);
831 } else {
832 ParallelTaskTerminator terminator(nworkers, traversal_gc->task_queues());
833 ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
834 _workers->run_task(&proc_task_proxy);
835 }
836 }
837
838 void execute(EnqueueTask& task) {
839 ShenandoahTraversalRefEnqueueTaskProxy enqueue_task_proxy(task);
840 _workers->run_task(&enqueue_task_proxy);
841 }
842 };
843
844 void ShenandoahTraversalGC::weak_refs_work_doit() {
845 ShenandoahHeap* sh = ShenandoahHeap::heap();
846
847 ReferenceProcessor* rp = sh->ref_processor();
848
849 ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process;
850 ShenandoahPhaseTimings::Phase phase_enqueue = ShenandoahPhaseTimings::weakrefs_enqueue;
851
852 ReferenceProcessorIsAliveMutator fix_alive(rp, sh->is_alive_closure());
853
854 WorkGang* workers = sh->workers();
855 uint nworkers = workers->active_workers();
856
857 // Setup collector policy for softref cleaning.
858 bool clear_soft_refs = sh->collector_policy()->use_should_clear_all_soft_refs(true /* bogus arg*/);
859 log_develop_debug(gc, ref)("clearing soft refs: %s", BOOL_TO_STR(clear_soft_refs));
860 rp->setup_policy(clear_soft_refs);
861 rp->set_active_mt_degree(nworkers);
862
863 assert(task_queues()->is_empty(), "Should be empty");
864
865 // complete_gc and keep_alive closures instantiated here are only needed for
866 // single-threaded path in RP. They share the queue 0 for tracking work, which
867 // simplifies implementation. Since RP may decide to call complete_gc several
868 // times, we need to be able to reuse the terminator.
869 uint serial_worker_id = 0;
870 ParallelTaskTerminator terminator(1, task_queues());
871 ShenandoahTraversalDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
872
873 ShenandoahTraversalRefProcTaskExecutor executor(workers);
874
875 ReferenceProcessorPhaseTimes pt(sh->gc_timer(), rp->num_q());
876
877 {
878 ShenandoahGCPhase phase(phase_process);
879
880 ShenandoahForwardedIsAliveClosure is_alive;
881 ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id));
882 rp->process_discovered_references(&is_alive, &keep_alive,
883 &complete_gc, &executor,
884 &pt);
885 pt.print_all_references();
886
887 WeakProcessor::weak_oops_do(&is_alive, &keep_alive);
888
889 assert(!_heap->cancelled_concgc() || task_queues()->is_empty(), "Should be empty");
890 }
891
892 if (_heap->cancelled_concgc()) return;
893
894 {
895 ShenandoahGCPhase phase(phase_enqueue);
896 rp->enqueue_discovered_references(&executor, &pt);
897 pt.print_enqueue_phase();
898 }
899 }