1 /*
2 * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "classfile/symbolTable.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "code/codeCache.hpp"
30
31 #include "gc/shared/weakProcessor.inline.hpp"
32 #include "gc/shared/gcTimer.hpp"
33 #include "gc/shared/gcTrace.hpp"
34 #include "gc/shared/referenceProcessor.hpp"
35 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
36 #include "gc/shared/strongRootsScope.hpp"
37
38 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
39 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
42 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
43 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
44 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
45 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
46 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
47 #include "gc/shenandoah/shenandoahUtils.hpp"
48
49 #include "memory/iterator.inline.hpp"
50 #include "memory/metaspace.hpp"
51 #include "memory/resourceArea.hpp"
52 #include "oops/oop.inline.hpp"
53 #include "runtime/handles.inline.hpp"
54
55 template<UpdateRefsMode UPDATE_REFS>
56 class ShenandoahInitMarkRootsClosure : public OopClosure {
57 private:
58 ShenandoahObjToScanQueue* _queue;
59 ShenandoahHeap* _heap;
60 ShenandoahMarkingContext* const _mark_context;
61
62 template <class T>
63 inline void do_oop_work(T* p) {
64 ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, NO_DEDUP>(p, _heap, _queue, _mark_context);
65 }
66
67 public:
68 ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) :
69 _queue(q),
70 _heap(ShenandoahHeap::heap()),
71 _mark_context(_heap->marking_context()) {};
72
73 void do_oop(narrowOop* p) { do_oop_work(p); }
74 void do_oop(oop* p) { do_oop_work(p); }
75 };
76
77 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
78 MetadataVisitingOopIterateClosure(rp),
79 _queue(q),
80 _heap(ShenandoahHeap::heap()),
81 _mark_context(_heap->marking_context())
82 { }
83
84 template<UpdateRefsMode UPDATE_REFS>
85 class ShenandoahInitMarkRootsTask : public AbstractGangTask {
86 private:
87 ShenandoahAllRootScanner* _rp;
88 public:
89 ShenandoahInitMarkRootsTask(ShenandoahAllRootScanner* rp) :
90 AbstractGangTask("Shenandoah init mark roots task"),
91 _rp(rp) {
92 }
93
94 void work(uint worker_id) {
95 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
96 ShenandoahParallelWorkerSession worker_session(worker_id);
97
98 ShenandoahHeap* heap = ShenandoahHeap::heap();
99 ShenandoahObjToScanQueueSet* queues = heap->concurrent_mark()->task_queues();
100 assert(queues->get_reserved() > worker_id, "Queue has not been reserved for worker id: %d", worker_id);
101
102 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
103
104 ShenandoahInitMarkRootsClosure<UPDATE_REFS> mark_cl(q);
105 do_work(heap, &mark_cl, worker_id);
106 }
107
108 private:
109 void do_work(ShenandoahHeap* heap, OopClosure* oops, uint worker_id) {
110 // The rationale for selecting the roots to scan is as follows:
111 // a. With unload_classes = true, we only want to scan the actual strong roots from the
112 // code cache. This will allow us to identify the dead classes, unload them, *and*
113 // invalidate the relevant code cache blobs. This could be only done together with
114 // class unloading.
115 // b. With unload_classes = false, we have to nominally retain all the references from code
116 // cache, because there could be the case of embedded class/oop in the generated code,
117 // which we will never visit during mark. Without code cache invalidation, as in (a),
118 // we risk executing that code cache blob, and crashing.
119 if (heap->unload_classes()) {
120 _rp->strong_roots_do(worker_id, oops);
121 } else {
122 _rp->roots_do(worker_id, oops);
123 }
124 }
125 };
126
127 class ShenandoahUpdateRootsTask : public AbstractGangTask {
128 private:
129 ShenandoahRootUpdater* _root_updater;
130 bool _check_alive;
131 public:
132 ShenandoahUpdateRootsTask(ShenandoahRootUpdater* root_updater, bool check_alive) :
133 AbstractGangTask("Shenandoah update roots task"),
134 _root_updater(root_updater),
135 _check_alive(check_alive){
136 }
137
138 void work(uint worker_id) {
139 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
140 ShenandoahParallelWorkerSession worker_session(worker_id);
141
142 ShenandoahHeap* heap = ShenandoahHeap::heap();
143 ShenandoahUpdateRefsClosure cl;
144 if (_check_alive) {
145 ShenandoahForwardedIsAliveClosure is_alive;
146 _root_updater->roots_do<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>(worker_id, &is_alive, &cl);
147 } else {
148 AlwaysTrueClosure always_true;;
149 _root_updater->roots_do<AlwaysTrueClosure, ShenandoahUpdateRefsClosure>(worker_id, &always_true, &cl);
150 }
151 }
152 };
153
154 class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
155 private:
156 ShenandoahConcurrentMark* _cm;
157 TaskTerminator* _terminator;
158
159 public:
160 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
161 AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator) {
162 }
163
164 void work(uint worker_id) {
165 ShenandoahHeap* heap = ShenandoahHeap::heap();
166 ShenandoahConcurrentWorkerSession worker_session(worker_id);
167 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
168 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
169 ReferenceProcessor* rp;
170 if (heap->process_references()) {
171 rp = heap->ref_processor();
172 shenandoah_assert_rp_isalive_installed();
173 } else {
174 rp = NULL;
175 }
176
177 _cm->concurrent_scan_code_roots(worker_id, rp);
178 _cm->mark_loop(worker_id, _terminator, rp,
179 true, // cancellable
180 ShenandoahStringDedup::is_enabled()); // perform string dedup
181 }
182 };
183
184 class ShenandoahSATBAndRemarkCodeRootsThreadsClosure : public ThreadClosure {
185 private:
186 ShenandoahSATBBufferClosure* _satb_cl;
187 OopClosure* const _cl;
188 MarkingCodeBlobClosure* _code_cl;
189 uintx _claim_token;
190
191 public:
192 ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, OopClosure* cl, MarkingCodeBlobClosure* code_cl) :
193 _satb_cl(satb_cl), _cl(cl), _code_cl(code_cl),
194 _claim_token(Threads::thread_claim_token()) {}
195
196 void do_thread(Thread* thread) {
197 if (thread->claim_threads_do(true, _claim_token)) {
198 ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
199 if (thread->is_Java_thread()) {
200 if (_cl != NULL) {
201 ResourceMark rm;
202 thread->oops_do(_cl, _code_cl);
203 } else if (_code_cl != NULL) {
204 // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
205 // however the liveness of oops reachable from nmethods have very complex lifecycles:
206 // * Alive if on the stack of an executing method
207 // * Weakly reachable otherwise
208 // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
209 // live by the SATB invariant but other oops recorded in nmethods may behave differently.
210 JavaThread* jt = (JavaThread*)thread;
211 jt->nmethods_do(_code_cl);
212 }
213 }
214 }
215 }
216 };
217
218 class ShenandoahFinalMarkingTask : public AbstractGangTask {
219 private:
220 ShenandoahConcurrentMark* _cm;
221 TaskTerminator* _terminator;
222 bool _dedup_string;
223
224 public:
225 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
226 AbstractGangTask("Shenandoah Final Marking"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
227 }
228
229 void work(uint worker_id) {
230 ShenandoahHeap* heap = ShenandoahHeap::heap();
231
232 ShenandoahParallelWorkerSession worker_session(worker_id);
233 ReferenceProcessor* rp;
234 if (heap->process_references()) {
235 rp = heap->ref_processor();
236 shenandoah_assert_rp_isalive_installed();
237 } else {
238 rp = NULL;
239 }
240
241 // First drain remaining SATB buffers.
242 // Notice that this is not strictly necessary for mark-compact. But since
243 // it requires a StrongRootsScope around the task, we need to claim the
244 // threads, and performance-wise it doesn't really matter. Adds about 1ms to
245 // full-gc.
246 {
247 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
248
249 ShenandoahSATBBufferClosure cl(q);
250 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
251 while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
252 bool do_nmethods = heap->unload_classes() && !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading();
253 if (heap->has_forwarded_objects()) {
254 ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp);
255 MarkingCodeBlobClosure blobsCl(&resolve_mark_cl, !CodeBlobToOopClosure::FixRelocations);
256 ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
257 ShenandoahStoreValEnqueueBarrier ? &resolve_mark_cl : NULL,
258 do_nmethods ? &blobsCl : NULL);
259 Threads::threads_do(&tc);
260 } else {
261 ShenandoahMarkRefsClosure mark_cl(q, rp);
262 MarkingCodeBlobClosure blobsCl(&mark_cl, !CodeBlobToOopClosure::FixRelocations);
263 ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
264 ShenandoahStoreValEnqueueBarrier ? &mark_cl : NULL,
265 do_nmethods ? &blobsCl : NULL);
266 Threads::threads_do(&tc);
267 }
268 }
269
270 if (heap->is_degenerated_gc_in_progress()) {
271 // Degenerated cycle may bypass concurrent cycle, so code roots might not be scanned,
272 // let's check here.
273 _cm->concurrent_scan_code_roots(worker_id, rp);
274 }
275
276 _cm->mark_loop(worker_id, _terminator, rp,
277 false, // not cancellable
278 _dedup_string);
279
280 assert(_cm->task_queues()->is_empty(), "Should be empty");
281 }
282 };
283
284 void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_phase) {
285 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
286 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
287
288 ShenandoahHeap* heap = ShenandoahHeap::heap();
289
290 ShenandoahGCPhase phase(root_phase);
291
292 WorkGang* workers = heap->workers();
293 uint nworkers = workers->active_workers();
294
295 assert(nworkers <= task_queues()->size(), "Just check");
296
297 ShenandoahAllRootScanner root_proc(nworkers, root_phase);
298 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
299 task_queues()->reserve(nworkers);
300
301 if (heap->has_forwarded_objects()) {
302 ShenandoahInitMarkRootsTask<RESOLVE> mark_roots(&root_proc);
303 workers->run_task(&mark_roots);
304 } else {
305 // No need to update references, which means the heap is stable.
306 // Can save time not walking through forwarding pointers.
307 ShenandoahInitMarkRootsTask<NONE> mark_roots(&root_proc);
308 workers->run_task(&mark_roots);
309 }
310
311 if (ShenandoahConcurrentScanCodeRoots) {
312 clear_claim_codecache();
313 }
314 }
315
316 void ShenandoahConcurrentMark::update_roots(ShenandoahPhaseTimings::Phase root_phase) {
317 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
318 assert(root_phase == ShenandoahPhaseTimings::full_gc_update_roots ||
319 root_phase == ShenandoahPhaseTimings::degen_gc_update_roots,
320 "Only for these phases");
321
322 ShenandoahGCPhase phase(root_phase);
323
324 bool check_alive = root_phase == ShenandoahPhaseTimings::degen_gc_update_roots;
325
326 #if COMPILER2_OR_JVMCI
327 DerivedPointerTable::clear();
328 #endif
329
330 uint nworkers = _heap->workers()->active_workers();
331
332 ShenandoahRootUpdater root_updater(nworkers, root_phase);
333 ShenandoahUpdateRootsTask update_roots(&root_updater, check_alive);
334 _heap->workers()->run_task(&update_roots);
335
336 #if COMPILER2_OR_JVMCI
337 DerivedPointerTable::update_pointers();
338 #endif
339 }
340
341 class ShenandoahUpdateThreadRootsTask : public AbstractGangTask {
342 private:
343 ShenandoahThreadRoots _thread_roots;
344 ShenandoahPhaseTimings::Phase _phase;
345 ShenandoahGCWorkerPhase _worker_phase;
346 public:
347 ShenandoahUpdateThreadRootsTask(bool is_par, ShenandoahPhaseTimings::Phase phase) :
348 AbstractGangTask("Shenandoah Update Thread Roots"),
349 _thread_roots(phase, is_par),
350 _phase(phase),
351 _worker_phase(phase) {}
352
353 void work(uint worker_id) {
354 ShenandoahParallelWorkerSession worker_session(worker_id);
355 ShenandoahUpdateRefsClosure cl;
356 _thread_roots.oops_do(&cl, NULL, worker_id);
357 }
358 };
359
360 void ShenandoahConcurrentMark::update_thread_roots(ShenandoahPhaseTimings::Phase root_phase) {
361 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
362
363 ShenandoahGCPhase phase(root_phase);
364
365 #if COMPILER2_OR_JVMCI
366 DerivedPointerTable::clear();
367 #endif
368
369 WorkGang* workers = _heap->workers();
370 bool is_par = workers->active_workers() > 1;
371
372 ShenandoahUpdateThreadRootsTask task(is_par, root_phase);
373 workers->run_task(&task);
374
375 #if COMPILER2_OR_JVMCI
376 DerivedPointerTable::update_pointers();
377 #endif
378 }
379
380 void ShenandoahConcurrentMark::initialize(uint workers) {
381 _heap = ShenandoahHeap::heap();
382
383 uint num_queues = MAX2(workers, 1U);
384
385 _task_queues = new ShenandoahObjToScanQueueSet((int) num_queues);
386
387 for (uint i = 0; i < num_queues; ++i) {
388 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
389 task_queue->initialize();
390 _task_queues->register_queue(i, task_queue);
391 }
392 }
393
394 void ShenandoahConcurrentMark::concurrent_scan_code_roots(uint worker_id, ReferenceProcessor* rp) {
395 if (ShenandoahConcurrentScanCodeRoots && claim_codecache()) {
396 ShenandoahObjToScanQueue* q = task_queues()->queue(worker_id);
397 if (!_heap->unload_classes()) {
398 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
399 // TODO: We can not honor StringDeduplication here, due to lock ranking
400 // inversion. So, we may miss some deduplication candidates.
401 if (_heap->has_forwarded_objects()) {
402 ShenandoahMarkResolveRefsClosure cl(q, rp);
403 CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
404 CodeCache::blobs_do(&blobs);
405 } else {
406 ShenandoahMarkRefsClosure cl(q, rp);
407 CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
408 CodeCache::blobs_do(&blobs);
409 }
410 }
411 }
412 }
413
414 void ShenandoahConcurrentMark::mark_from_roots() {
415 WorkGang* workers = _heap->workers();
416 uint nworkers = workers->active_workers();
417
418 if (_heap->process_references()) {
419 ReferenceProcessor* rp = _heap->ref_processor();
420 rp->set_active_mt_degree(nworkers);
421
422 // enable ("weak") refs discovery
423 rp->enable_discovery(true /*verify_no_refs*/);
424 rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
425 }
426
427 shenandoah_assert_rp_isalive_not_installed();
428 ShenandoahIsAliveSelector is_alive;
429 ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
430
431 task_queues()->reserve(nworkers);
432
433 {
434 TaskTerminator terminator(nworkers, task_queues());
435 ShenandoahConcurrentMarkingTask task(this, &terminator);
436 workers->run_task(&task);
437 }
438
439 assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty when not cancelled");
440 }
441
442 void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) {
443 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
444
445 uint nworkers = _heap->workers()->active_workers();
446
447 // Finally mark everything else we've got in our queues during the previous steps.
448 // It does two different things for concurrent vs. mark-compact GC:
449 // - For concurrent GC, it starts with empty task queues, drains the remaining
450 // SATB buffers, and then completes the marking closure.
451 // - For mark-compact GC, it starts out with the task queues seeded by initial
452 // root scan, and completes the closure, thus marking through all live objects
453 // The implementation is the same, so it's shared here.
454 {
455 ShenandoahGCPhase phase(full_gc ?
456 ShenandoahPhaseTimings::full_gc_mark_finish_queues :
457 ShenandoahPhaseTimings::finish_queues);
458 task_queues()->reserve(nworkers);
459
460 shenandoah_assert_rp_isalive_not_installed();
461 ShenandoahIsAliveSelector is_alive;
462 ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
463
464 StrongRootsScope scope(nworkers);
465 TaskTerminator terminator(nworkers, task_queues());
466 ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
467 _heap->workers()->run_task(&task);
468 }
469
470 assert(task_queues()->is_empty(), "Should be empty");
471
472 // When we're done marking everything, we process weak references.
473 if (_heap->process_references()) {
474 weak_refs_work(full_gc);
475 }
476
477 assert(task_queues()->is_empty(), "Should be empty");
478 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
479 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
480 }
481
482 // Weak Reference Closures
483 class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
484 uint _worker_id;
485 TaskTerminator* _terminator;
486 bool _reset_terminator;
487
488 public:
489 ShenandoahCMDrainMarkingStackClosure(uint worker_id, TaskTerminator* t, bool reset_terminator = false):
490 _worker_id(worker_id),
491 _terminator(t),
492 _reset_terminator(reset_terminator) {
493 }
494
495 void do_void() {
496 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
497
498 ShenandoahHeap* sh = ShenandoahHeap::heap();
499 ShenandoahConcurrentMark* scm = sh->concurrent_mark();
500 assert(sh->process_references(), "why else would we be here?");
501 ReferenceProcessor* rp = sh->ref_processor();
502
503 shenandoah_assert_rp_isalive_installed();
504
505 scm->mark_loop(_worker_id, _terminator, rp,
506 false, // not cancellable
507 false); // do not do strdedup
508
509 if (_reset_terminator) {
510 _terminator->reset_for_reuse();
511 }
512 }
513 };
514
515 class ShenandoahCMKeepAliveClosure : public OopClosure {
516 private:
517 ShenandoahObjToScanQueue* _queue;
518 ShenandoahHeap* _heap;
519 ShenandoahMarkingContext* const _mark_context;
520
521 template <class T>
522 inline void do_oop_work(T* p) {
523 ShenandoahConcurrentMark::mark_through_ref<T, NONE, NO_DEDUP>(p, _heap, _queue, _mark_context);
524 }
525
526 public:
527 ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) :
528 _queue(q),
529 _heap(ShenandoahHeap::heap()),
530 _mark_context(_heap->marking_context()) {}
531
532 void do_oop(narrowOop* p) { do_oop_work(p); }
533 void do_oop(oop* p) { do_oop_work(p); }
534 };
535
536 class ShenandoahCMKeepAliveUpdateClosure : public OopClosure {
537 private:
538 ShenandoahObjToScanQueue* _queue;
539 ShenandoahHeap* _heap;
540 ShenandoahMarkingContext* const _mark_context;
541
542 template <class T>
543 inline void do_oop_work(T* p) {
544 ShenandoahConcurrentMark::mark_through_ref<T, SIMPLE, NO_DEDUP>(p, _heap, _queue, _mark_context);
545 }
546
547 public:
548 ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
549 _queue(q),
550 _heap(ShenandoahHeap::heap()),
551 _mark_context(_heap->marking_context()) {}
552
553 void do_oop(narrowOop* p) { do_oop_work(p); }
554 void do_oop(oop* p) { do_oop_work(p); }
555 };
556
557 class ShenandoahWeakUpdateClosure : public OopClosure {
558 private:
559 ShenandoahHeap* const _heap;
560
561 template <class T>
562 inline void do_oop_work(T* p) {
563 oop o = _heap->maybe_update_with_forwarded(p);
564 shenandoah_assert_marked_except(p, o, o == NULL);
565 }
566
567 public:
568 ShenandoahWeakUpdateClosure() : _heap(ShenandoahHeap::heap()) {}
569
570 void do_oop(narrowOop* p) { do_oop_work(p); }
571 void do_oop(oop* p) { do_oop_work(p); }
572 };
573
574 class ShenandoahRefProcTaskProxy : public AbstractGangTask {
575 private:
576 AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
577 TaskTerminator* _terminator;
578
579 public:
580 ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
581 TaskTerminator* t) :
582 AbstractGangTask("Process reference objects in parallel"),
583 _proc_task(proc_task),
584 _terminator(t) {
585 }
586
587 void work(uint worker_id) {
588 ResourceMark rm;
589 HandleMark hm;
590 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
591 ShenandoahHeap* heap = ShenandoahHeap::heap();
592 ShenandoahParallelWorkerSession worker_session(worker_id);
593 ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator);
594 if (heap->has_forwarded_objects()) {
595 ShenandoahForwardedIsAliveClosure is_alive;
596 ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
597 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
598 } else {
599 ShenandoahIsAliveClosure is_alive;
600 ShenandoahCMKeepAliveClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
601 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
602 }
603 }
604 };
605
606 class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
607 private:
608 WorkGang* _workers;
609
610 public:
611 ShenandoahRefProcTaskExecutor(WorkGang* workers) :
612 _workers(workers) {
613 }
614
615 // Executes a task using worker threads.
616 void execute(ProcessTask& task, uint ergo_workers) {
617 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
618
619 ShenandoahHeap* heap = ShenandoahHeap::heap();
620 ShenandoahConcurrentMark* cm = heap->concurrent_mark();
621 ShenandoahPushWorkerQueuesScope scope(_workers, cm->task_queues(),
622 ergo_workers,
623 /* do_check = */ false);
624 uint nworkers = _workers->active_workers();
625 cm->task_queues()->reserve(nworkers);
626 TaskTerminator terminator(nworkers, cm->task_queues());
627 ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
628 _workers->run_task(&proc_task_proxy);
629 }
630 };
631
632 void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) {
633 assert(_heap->process_references(), "sanity");
634
635 ShenandoahPhaseTimings::Phase phase_root =
636 full_gc ?
637 ShenandoahPhaseTimings::full_gc_weakrefs :
638 ShenandoahPhaseTimings::weakrefs;
639
640 ShenandoahGCPhase phase(phase_root);
641
642 ReferenceProcessor* rp = _heap->ref_processor();
643
644 // NOTE: We cannot shortcut on has_discovered_references() here, because
645 // we will miss marking JNI Weak refs then, see implementation in
646 // ReferenceProcessor::process_discovered_references.
647 weak_refs_work_doit(full_gc);
648
649 rp->verify_no_references_recorded();
650 assert(!rp->discovery_enabled(), "Post condition");
651
652 }
653
654 void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) {
655 ReferenceProcessor* rp = _heap->ref_processor();
656
657 ShenandoahPhaseTimings::Phase phase_process =
658 full_gc ?
659 ShenandoahPhaseTimings::full_gc_weakrefs_process :
660 ShenandoahPhaseTimings::weakrefs_process;
661
662 shenandoah_assert_rp_isalive_not_installed();
663 ShenandoahIsAliveSelector is_alive;
664 ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
665
666 WorkGang* workers = _heap->workers();
667 uint nworkers = workers->active_workers();
668
669 rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
670 rp->set_active_mt_degree(nworkers);
671
672 assert(task_queues()->is_empty(), "Should be empty");
673
674 // complete_gc and keep_alive closures instantiated here are only needed for
675 // single-threaded path in RP. They share the queue 0 for tracking work, which
676 // simplifies implementation. Since RP may decide to call complete_gc several
677 // times, we need to be able to reuse the terminator.
678 uint serial_worker_id = 0;
679 TaskTerminator terminator(1, task_queues());
680 ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
681
682 ShenandoahRefProcTaskExecutor executor(workers);
683
684 ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
685
686 {
687 // Note: Don't emit JFR event for this phase, to avoid overflow nesting phase level.
688 // Reference Processor emits 2 levels JFR event, that can get us over the JFR
689 // event nesting level limits, in case of degenerated GC gets upgraded to
690 // full GC.
691 ShenandoahGCPhaseTiming phase_timing(phase_process);
692
693 if (_heap->has_forwarded_objects()) {
694 ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id));
695 const ReferenceProcessorStats& stats =
696 rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
697 &complete_gc, &executor,
698 &pt);
699 _heap->tracer()->report_gc_reference_stats(stats);
700 } else {
701 ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id));
702 const ReferenceProcessorStats& stats =
703 rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
704 &complete_gc, &executor,
705 &pt);
706 _heap->tracer()->report_gc_reference_stats(stats);
707 }
708
709 pt.print_all_references();
710
711 assert(task_queues()->is_empty(), "Should be empty");
712 }
713 }
714
715 class ShenandoahCancelledGCYieldClosure : public YieldClosure {
716 private:
717 ShenandoahHeap* const _heap;
718 public:
719 ShenandoahCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
720 virtual bool should_return() { return _heap->cancelled_gc(); }
721 };
722
723 class ShenandoahPrecleanCompleteGCClosure : public VoidClosure {
724 public:
725 void do_void() {
726 ShenandoahHeap* sh = ShenandoahHeap::heap();
727 ShenandoahConcurrentMark* scm = sh->concurrent_mark();
728 assert(sh->process_references(), "why else would we be here?");
729 TaskTerminator terminator(1, scm->task_queues());
730
731 ReferenceProcessor* rp = sh->ref_processor();
732 shenandoah_assert_rp_isalive_installed();
733
734 scm->mark_loop(0, &terminator, rp,
735 false, // not cancellable
736 false); // do not do strdedup
737 }
738 };
739
740 class ShenandoahPrecleanTask : public AbstractGangTask {
741 private:
742 ReferenceProcessor* _rp;
743
744 public:
745 ShenandoahPrecleanTask(ReferenceProcessor* rp) :
746 AbstractGangTask("Precleaning task"),
747 _rp(rp) {}
748
749 void work(uint worker_id) {
750 assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
751 ShenandoahParallelWorkerSession worker_session(worker_id);
752
753 ShenandoahHeap* sh = ShenandoahHeap::heap();
754 assert(!sh->has_forwarded_objects(), "No forwarded objects expected here");
755
756 ShenandoahObjToScanQueue* q = sh->concurrent_mark()->get_queue(worker_id);
757
758 ShenandoahCancelledGCYieldClosure yield;
759 ShenandoahPrecleanCompleteGCClosure complete_gc;
760
761 ShenandoahIsAliveClosure is_alive;
762 ShenandoahCMKeepAliveClosure keep_alive(q);
763 ResourceMark rm;
764 _rp->preclean_discovered_references(&is_alive, &keep_alive,
765 &complete_gc, &yield,
766 NULL);
767 }
768 };
769
770 void ShenandoahConcurrentMark::preclean_weak_refs() {
771 // Pre-cleaning weak references before diving into STW makes sense at the
772 // end of concurrent mark. This will filter out the references which referents
773 // are alive. Note that ReferenceProcessor already filters out these on reference
774 // discovery, and the bulk of work is done here. This phase processes leftovers
775 // that missed the initial filtering, i.e. when referent was marked alive after
776 // reference was discovered by RP.
777
778 assert(_heap->process_references(), "sanity");
779
780 // Shortcut if no references were discovered to avoid winding up threads.
781 ReferenceProcessor* rp = _heap->ref_processor();
782 if (!rp->has_discovered_references()) {
783 return;
784 }
785
786 assert(task_queues()->is_empty(), "Should be empty");
787
788 ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
789
790 shenandoah_assert_rp_isalive_not_installed();
791 ShenandoahIsAliveSelector is_alive;
792 ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
793
794 // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
795 // queues and other goodies. When upstream ReferenceProcessor starts supporting
796 // parallel precleans, we can extend this to more threads.
797 WorkGang* workers = _heap->workers();
798 uint nworkers = workers->active_workers();
799 assert(nworkers == 1, "This code uses only a single worker");
800 task_queues()->reserve(nworkers);
801
802 ShenandoahPrecleanTask task(rp);
803 workers->run_task(&task);
804
805 assert(task_queues()->is_empty(), "Should be empty");
806 }
807
808 void ShenandoahConcurrentMark::cancel() {
809 // Clean up marking stacks.
810 ShenandoahObjToScanQueueSet* queues = task_queues();
811 queues->clear();
812
813 // Cancel SATB buffers.
814 ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
815 }
816
817 ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
818 assert(task_queues()->get_reserved() > worker_id, "No reserved queue for worker id: %d", worker_id);
819 return _task_queues->queue(worker_id);
820 }
821
822 template <bool CANCELLABLE>
823 void ShenandoahConcurrentMark::mark_loop_prework(uint w, TaskTerminator *t, ReferenceProcessor *rp,
824 bool strdedup) {
825 ShenandoahObjToScanQueue* q = get_queue(w);
826
827 ShenandoahLiveData* ld = _heap->get_liveness_cache(w);
828
829 // TODO: We can clean up this if we figure out how to do templated oop closures that
830 // play nice with specialized_oop_iterators.
831 if (_heap->unload_classes()) {
832 if (_heap->has_forwarded_objects()) {
833 if (strdedup) {
834 ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp);
835 mark_loop_work<ShenandoahMarkUpdateRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
836 } else {
837 ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp);
838 mark_loop_work<ShenandoahMarkUpdateRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
839 }
840 } else {
841 if (strdedup) {
842 ShenandoahMarkRefsMetadataDedupClosure cl(q, rp);
843 mark_loop_work<ShenandoahMarkRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
844 } else {
845 ShenandoahMarkRefsMetadataClosure cl(q, rp);
846 mark_loop_work<ShenandoahMarkRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
847 }
848 }
849 } else {
850 if (_heap->has_forwarded_objects()) {
851 if (strdedup) {
852 ShenandoahMarkUpdateRefsDedupClosure cl(q, rp);
853 mark_loop_work<ShenandoahMarkUpdateRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
854 } else {
855 ShenandoahMarkUpdateRefsClosure cl(q, rp);
856 mark_loop_work<ShenandoahMarkUpdateRefsClosure, CANCELLABLE>(&cl, ld, w, t);
857 }
858 } else {
859 if (strdedup) {
860 ShenandoahMarkRefsDedupClosure cl(q, rp);
861 mark_loop_work<ShenandoahMarkRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
862 } else {
863 ShenandoahMarkRefsClosure cl(q, rp);
864 mark_loop_work<ShenandoahMarkRefsClosure, CANCELLABLE>(&cl, ld, w, t);
865 }
866 }
867 }
868
869 _heap->flush_liveness_cache(w);
870 }
871
872 template <class T, bool CANCELLABLE>
873 void ShenandoahConcurrentMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *terminator) {
874 uintx stride = ShenandoahMarkLoopStride;
875
876 ShenandoahHeap* heap = ShenandoahHeap::heap();
877 ShenandoahObjToScanQueueSet* queues = task_queues();
878 ShenandoahObjToScanQueue* q;
879 ShenandoahMarkTask t;
880
881 /*
882 * Process outstanding queues, if any.
883 *
884 * There can be more queues than workers. To deal with the imbalance, we claim
885 * extra queues first. Since marking can push new tasks into the queue associated
886 * with this worker id, we come back to process this queue in the normal loop.
887 */
888 assert(queues->get_reserved() == heap->workers()->active_workers(),
889 "Need to reserve proper number of queues: reserved: %u, active: %u", queues->get_reserved(), heap->workers()->active_workers());
890
891 q = queues->claim_next();
892 while (q != NULL) {
893 if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
894 return;
895 }
896
897 for (uint i = 0; i < stride; i++) {
898 if (q->pop(t)) {
899 do_task<T>(q, cl, live_data, &t);
900 } else {
901 assert(q->is_empty(), "Must be empty");
902 q = queues->claim_next();
903 break;
904 }
905 }
906 }
907 q = get_queue(worker_id);
908
909 ShenandoahSATBBufferClosure drain_satb(q);
910 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
911
912 /*
913 * Normal marking loop:
914 */
915 while (true) {
916 if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
917 return;
918 }
919
920 while (satb_mq_set.completed_buffers_num() > 0) {
921 satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
922 }
923
924 uint work = 0;
925 for (uint i = 0; i < stride; i++) {
926 if (q->pop(t) ||
927 queues->steal(worker_id, t)) {
928 do_task<T>(q, cl, live_data, &t);
929 work++;
930 } else {
931 break;
932 }
933 }
934
935 if (work == 0) {
936 // No work encountered in current stride, try to terminate.
937 // Need to leave the STS here otherwise it might block safepoints.
938 ShenandoahSuspendibleThreadSetLeaver stsl(CANCELLABLE && ShenandoahSuspendibleWorkers);
939 ShenandoahTerminatorTerminator tt(heap);
940 if (terminator->offer_termination(&tt)) return;
941 }
942 }
943 }
944
945 bool ShenandoahConcurrentMark::claim_codecache() {
946 assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise");
947 return _claimed_codecache.try_set();
948 }
949
950 void ShenandoahConcurrentMark::clear_claim_codecache() {
951 assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise");
952 _claimed_codecache.unset();
953 }