1 /*
2 * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "classfile/symbolTable.hpp"
27 #include "classfile/systemDictionary.hpp"
28 #include "code/codeCache.hpp"
29
30 #include "gc/shared/weakProcessor.inline.hpp"
31 #include "gc/shared/gcTimer.hpp"
32 #include "gc/shared/referenceProcessor.hpp"
33 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
34
35 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
36 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
37 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
38 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
39 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
40 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
41 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
42 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
43 #include "gc/shenandoah/shenandoahUtils.hpp"
44
45 #include "memory/iterator.inline.hpp"
46 #include "memory/metaspace.hpp"
47 #include "memory/resourceArea.hpp"
48 #include "oops/oop.inline.hpp"
49 #include "runtime/handles.inline.hpp"
50
51 template<UpdateRefsMode UPDATE_REFS, StringDedupMode STRING_DEDUP>
52 class ShenandoahInitMarkRootsClosure : public OopClosure {
53 private:
54 ShenandoahObjToScanQueue* _queue;
55 ShenandoahHeap* _heap;
56 ShenandoahMarkingContext* const _mark_context;
57
58 template <class T>
59 inline void do_oop_work(T* p) {
60 ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, STRING_DEDUP>(p, _heap, _queue, _mark_context);
61 }
62
63 public:
64 ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) :
65 _queue(q),
66 _heap(ShenandoahHeap::heap()),
67 _mark_context(_heap->marking_context()) {};
68
69 void do_oop(narrowOop* p) { do_oop_work(p); }
70 void do_oop(oop* p) { do_oop_work(p); }
71 };
72
73 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
74 MetadataVisitingOopIterateClosure(rp),
75 _queue(q),
76 _heap(ShenandoahHeap::heap()),
77 _mark_context(_heap->marking_context())
78 { }
79
80 template<UpdateRefsMode UPDATE_REFS>
81 class ShenandoahInitMarkRootsTask : public AbstractGangTask {
82 private:
83 ShenandoahRootProcessor* _rp;
84 bool _process_refs;
85 public:
86 ShenandoahInitMarkRootsTask(ShenandoahRootProcessor* rp, bool process_refs) :
87 AbstractGangTask("Shenandoah init mark roots task"),
88 _rp(rp),
89 _process_refs(process_refs) {
90 }
91
92 void work(uint worker_id) {
93 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
94 ShenandoahParallelWorkerSession worker_session(worker_id);
95
96 ShenandoahHeap* heap = ShenandoahHeap::heap();
97 ShenandoahObjToScanQueueSet* queues = heap->concurrent_mark()->task_queues();
98 assert(queues->get_reserved() > worker_id, "Queue has not been reserved for worker id: %d", worker_id);
99
100 ShenandoahObjToScanQueue* q = queues->queue(worker_id);
101
102 if (ShenandoahStringDedup::is_enabled()) {
103 ShenandoahInitMarkRootsClosure<UPDATE_REFS, ENQUEUE_DEDUP> mark_cl(q);
104 do_work(heap, &mark_cl, worker_id);
105 } else {
106 ShenandoahInitMarkRootsClosure<UPDATE_REFS, NO_DEDUP> mark_cl(q);
107 do_work(heap, &mark_cl, worker_id);
108 }
109 }
110
111 private:
112 void do_work(ShenandoahHeap* heap, OopClosure* oops, uint worker_id) {
113 // The rationale for selecting the roots to scan is as follows:
114 // a. With unload_classes = true, we only want to scan the actual strong roots from the
115 // code cache. This will allow us to identify the dead classes, unload them, *and*
116 // invalidate the relevant code cache blobs. This could be only done together with
117 // class unloading.
118 // b. With unload_classes = false, we have to nominally retain all the references from code
119 // cache, because there could be the case of embedded class/oop in the generated code,
120 // which we will never visit during mark. Without code cache invalidation, as in (a),
121 // we risk executing that code cache blob, and crashing.
122 // c. With ShenandoahConcurrentScanCodeRoots, we avoid scanning the entire code cache here,
123 // and instead do that in concurrent phase under the relevant lock. This saves init mark
124 // pause time.
125
126 CLDToOopClosure clds_cl(oops, ClassLoaderData::_claim_strong);
127 MarkingCodeBlobClosure blobs_cl(oops, ! CodeBlobToOopClosure::FixRelocations);
128 OopClosure* weak_oops = _process_refs ? NULL : oops;
129
130 ResourceMark m;
131 if (heap->unload_classes()) {
132 _rp->process_strong_roots(oops, weak_oops, &clds_cl, NULL, &blobs_cl, NULL, worker_id);
133 } else {
134 if (ShenandoahConcurrentScanCodeRoots) {
135 CodeBlobClosure* code_blobs = NULL;
136 #ifdef ASSERT
137 ShenandoahAssertToSpaceClosure assert_to_space_oops;
138 CodeBlobToOopClosure assert_to_space(&assert_to_space_oops, !CodeBlobToOopClosure::FixRelocations);
139 // If conc code cache evac is disabled, code cache should have only to-space ptrs.
140 // Otherwise, it should have to-space ptrs only if mark does not update refs.
141 if (!heap->has_forwarded_objects()) {
142 code_blobs = &assert_to_space;
143 }
144 #endif
145 _rp->process_all_roots(oops, weak_oops, &clds_cl, code_blobs, NULL, worker_id);
146 } else {
147 _rp->process_all_roots(oops, weak_oops, &clds_cl, &blobs_cl, NULL, worker_id);
148 }
149 }
150 }
151 };
152
153 class ShenandoahUpdateRootsTask : public AbstractGangTask {
154 private:
155 ShenandoahRootProcessor* _rp;
156 const bool _update_code_cache;
157 public:
158 ShenandoahUpdateRootsTask(ShenandoahRootProcessor* rp, bool update_code_cache) :
159 AbstractGangTask("Shenandoah update roots task"),
160 _rp(rp),
161 _update_code_cache(update_code_cache) {
162 }
163
164 void work(uint worker_id) {
165 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
166 ShenandoahParallelWorkerSession worker_session(worker_id);
167
168 ShenandoahHeap* heap = ShenandoahHeap::heap();
169 ShenandoahUpdateRefsClosure cl;
170 CLDToOopClosure cldCl(&cl, ClassLoaderData::_claim_strong);
171
172 CodeBlobClosure* code_blobs;
173 CodeBlobToOopClosure update_blobs(&cl, CodeBlobToOopClosure::FixRelocations);
174 #ifdef ASSERT
175 ShenandoahAssertToSpaceClosure assert_to_space_oops;
176 CodeBlobToOopClosure assert_to_space(&assert_to_space_oops, !CodeBlobToOopClosure::FixRelocations);
177 #endif
178 if (_update_code_cache) {
179 code_blobs = &update_blobs;
180 } else {
181 code_blobs =
182 DEBUG_ONLY(&assert_to_space)
183 NOT_DEBUG(NULL);
184 }
185 _rp->process_all_roots(&cl, &cl, &cldCl, code_blobs, NULL, worker_id);
186 }
187 };
188
189 class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
190 private:
191 ShenandoahConcurrentMark* _cm;
192 ShenandoahTaskTerminator* _terminator;
193
194 public:
195 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, ShenandoahTaskTerminator* terminator) :
196 AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator) {
197 }
198
199 void work(uint worker_id) {
200 ShenandoahHeap* heap = ShenandoahHeap::heap();
201 ShenandoahConcurrentWorkerSession worker_session(worker_id);
202 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
203 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
204 ReferenceProcessor* rp;
205 if (heap->process_references()) {
206 rp = heap->ref_processor();
207 shenandoah_assert_rp_isalive_installed();
208 } else {
209 rp = NULL;
210 }
211
212 _cm->concurrent_scan_code_roots(worker_id, rp);
213 _cm->mark_loop(worker_id, _terminator, rp,
214 true, // cancellable
215 ShenandoahStringDedup::is_enabled()); // perform string dedup
216 }
217 };
218
219 class ShenandoahSATBThreadsClosure : public ThreadClosure {
220 private:
221 ShenandoahSATBBufferClosure* _satb_cl;
222 int _thread_parity;
223
224 public:
225 ShenandoahSATBThreadsClosure(ShenandoahSATBBufferClosure* satb_cl) :
226 _satb_cl(satb_cl),
227 _thread_parity(Threads::thread_claim_parity()) {}
228
229 void do_thread(Thread* thread) {
230 if (thread->claim_oops_do(true, _thread_parity)) {
231 ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
232 }
233 }
234 };
235
236 class ShenandoahFinalMarkingTask : public AbstractGangTask {
237 private:
238 ShenandoahConcurrentMark* _cm;
239 ShenandoahTaskTerminator* _terminator;
240 bool _dedup_string;
241
242 public:
243 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, ShenandoahTaskTerminator* terminator, bool dedup_string) :
244 AbstractGangTask("Shenandoah Final Marking"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
245 }
246
247 void work(uint worker_id) {
248 ShenandoahHeap* heap = ShenandoahHeap::heap();
249
250 ShenandoahParallelWorkerSession worker_session(worker_id);
251 // First drain remaining SATB buffers.
252 // Notice that this is not strictly necessary for mark-compact. But since
253 // it requires a StrongRootsScope around the task, we need to claim the
254 // threads, and performance-wise it doesn't really matter. Adds about 1ms to
255 // full-gc.
256 {
257 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
258 ShenandoahSATBBufferClosure cl(q);
259 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
260 while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
261 ShenandoahSATBThreadsClosure tc(&cl);
262 Threads::threads_do(&tc);
263 }
264
265 ReferenceProcessor* rp;
266 if (heap->process_references()) {
267 rp = heap->ref_processor();
268 shenandoah_assert_rp_isalive_installed();
269 } else {
270 rp = NULL;
271 }
272
273 // Degenerated cycle may bypass concurrent cycle, so code roots might not be scanned,
274 // let's check here.
275 _cm->concurrent_scan_code_roots(worker_id, rp);
276 _cm->mark_loop(worker_id, _terminator, rp,
277 false, // not cancellable
278 _dedup_string);
279
280 assert(_cm->task_queues()->is_empty(), "Should be empty");
281 }
282 };
283
284 void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_phase) {
285 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
286 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
287
288 ShenandoahHeap* heap = ShenandoahHeap::heap();
289
290 ShenandoahGCPhase phase(root_phase);
291
292 WorkGang* workers = heap->workers();
293 uint nworkers = workers->active_workers();
294
295 assert(nworkers <= task_queues()->size(), "Just check");
296
297 ShenandoahRootProcessor root_proc(heap, nworkers, root_phase);
298 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
299 task_queues()->reserve(nworkers);
300
301 if (heap->has_forwarded_objects()) {
302 ShenandoahInitMarkRootsTask<RESOLVE> mark_roots(&root_proc, _heap->process_references());
303 workers->run_task(&mark_roots);
304 } else {
305 // No need to update references, which means the heap is stable.
306 // Can save time not walking through forwarding pointers.
307 ShenandoahInitMarkRootsTask<NONE> mark_roots(&root_proc, _heap->process_references());
308 workers->run_task(&mark_roots);
309 }
310
311 if (ShenandoahConcurrentScanCodeRoots) {
312 clear_claim_codecache();
313 }
314 }
315
316 void ShenandoahConcurrentMark::update_roots(ShenandoahPhaseTimings::Phase root_phase) {
317 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
318
319 bool update_code_cache = true; // initialize to safer value
320 switch (root_phase) {
321 case ShenandoahPhaseTimings::update_roots:
322 case ShenandoahPhaseTimings::final_update_refs_roots:
323 update_code_cache = false;
324 break;
325 case ShenandoahPhaseTimings::full_gc_roots:
326 case ShenandoahPhaseTimings::degen_gc_update_roots:
327 update_code_cache = true;
328 break;
329 default:
330 ShouldNotReachHere();
331 }
332
333 ShenandoahGCPhase phase(root_phase);
334
335 #if defined(COMPILER2) || INCLUDE_JVMCI
336 DerivedPointerTable::clear();
337 #endif
338
339 uint nworkers = _heap->workers()->active_workers();
340
341 ShenandoahRootProcessor root_proc(_heap, nworkers, root_phase);
342 ShenandoahUpdateRootsTask update_roots(&root_proc, update_code_cache);
343 _heap->workers()->run_task(&update_roots);
344
345 #if defined(COMPILER2) || INCLUDE_JVMCI
346 DerivedPointerTable::update_pointers();
347 #endif
348 }
349
350 void ShenandoahConcurrentMark::initialize(uint workers) {
351 _heap = ShenandoahHeap::heap();
352
353 uint num_queues = MAX2(workers, 1U);
354
355 _task_queues = new ShenandoahObjToScanQueueSet((int) num_queues);
356
357 for (uint i = 0; i < num_queues; ++i) {
358 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
359 task_queue->initialize();
360 _task_queues->register_queue(i, task_queue);
361 }
362 }
363
364 void ShenandoahConcurrentMark::concurrent_scan_code_roots(uint worker_id, ReferenceProcessor* rp) {
365 if (ShenandoahConcurrentScanCodeRoots && claim_codecache()) {
366 ShenandoahObjToScanQueue* q = task_queues()->queue(worker_id);
367 if (!_heap->unload_classes()) {
368 MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
369 // TODO: We can not honor StringDeduplication here, due to lock ranking
370 // inversion. So, we may miss some deduplication candidates.
371 if (_heap->has_forwarded_objects()) {
372 ShenandoahMarkResolveRefsClosure cl(q, rp);
373 CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
374 CodeCache::blobs_do(&blobs);
375 } else {
376 ShenandoahMarkRefsClosure cl(q, rp);
377 CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
378 CodeCache::blobs_do(&blobs);
379 }
380 }
381 }
382 }
383
384 void ShenandoahConcurrentMark::mark_from_roots() {
385 WorkGang* workers = _heap->workers();
386 uint nworkers = workers->active_workers();
387
388 ShenandoahGCPhase conc_mark_phase(ShenandoahPhaseTimings::conc_mark);
389
390 if (_heap->process_references()) {
391 ReferenceProcessor* rp = _heap->ref_processor();
392 rp->set_active_mt_degree(nworkers);
393
394 // enable ("weak") refs discovery
395 rp->enable_discovery(true /*verify_no_refs*/);
396 rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
397 }
398
399 shenandoah_assert_rp_isalive_not_installed();
400 ShenandoahIsAliveSelector is_alive;
401 ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
402
403 task_queues()->reserve(nworkers);
404
405 {
406 ShenandoahTerminationTracker term(ShenandoahPhaseTimings::conc_termination);
407 ShenandoahTaskTerminator terminator(nworkers, task_queues());
408 ShenandoahConcurrentMarkingTask task(this, &terminator);
409 workers->run_task(&task);
410 }
411
412 assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty when not cancelled");
413 }
414
415 void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) {
416 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
417
418 uint nworkers = _heap->workers()->active_workers();
419
420 // Finally mark everything else we've got in our queues during the previous steps.
421 // It does two different things for concurrent vs. mark-compact GC:
422 // - For concurrent GC, it starts with empty task queues, drains the remaining
423 // SATB buffers, and then completes the marking closure.
424 // - For mark-compact GC, it starts out with the task queues seeded by initial
425 // root scan, and completes the closure, thus marking through all live objects
426 // The implementation is the same, so it's shared here.
427 {
428 ShenandoahGCPhase phase(full_gc ?
429 ShenandoahPhaseTimings::full_gc_mark_finish_queues :
430 ShenandoahPhaseTimings::finish_queues);
431 task_queues()->reserve(nworkers);
432
433 shenandoah_assert_rp_isalive_not_installed();
434 ShenandoahIsAliveSelector is_alive;
435 ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
436
437 ShenandoahTerminationTracker termination_tracker(full_gc ?
438 ShenandoahPhaseTimings::full_gc_mark_termination :
439 ShenandoahPhaseTimings::termination);
440
441 StrongRootsScope scope(nworkers);
442 ShenandoahTaskTerminator terminator(nworkers, task_queues());
443 ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
444 _heap->workers()->run_task(&task);
445 }
446
447 assert(task_queues()->is_empty(), "Should be empty");
448
449 // When we're done marking everything, we process weak references.
450 if (_heap->process_references()) {
451 weak_refs_work(full_gc);
452 }
453
454 // And finally finish class unloading
455 if (_heap->unload_classes()) {
456 _heap->unload_classes_and_cleanup_tables(full_gc);
457 }
458
459 assert(task_queues()->is_empty(), "Should be empty");
460 TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
461 TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
462
463 // Resize Metaspace
464 MetaspaceGC::compute_new_size();
465 }
466
467 // Weak Reference Closures
468 class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
469 uint _worker_id;
470 ShenandoahTaskTerminator* _terminator;
471 bool _reset_terminator;
472
473 public:
474 ShenandoahCMDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
475 _worker_id(worker_id),
476 _terminator(t),
477 _reset_terminator(reset_terminator) {
478 }
479
480 void do_void() {
481 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
482
483 ShenandoahHeap* sh = ShenandoahHeap::heap();
484 ShenandoahConcurrentMark* scm = sh->concurrent_mark();
485 assert(sh->process_references(), "why else would we be here?");
486 ReferenceProcessor* rp = sh->ref_processor();
487
488 shenandoah_assert_rp_isalive_installed();
489
490 scm->mark_loop(_worker_id, _terminator, rp,
491 false, // not cancellable
492 false); // do not do strdedup
493
494 if (_reset_terminator) {
495 _terminator->reset_for_reuse();
496 }
497 }
498 };
499
500 class ShenandoahCMKeepAliveClosure : public OopClosure {
501 private:
502 ShenandoahObjToScanQueue* _queue;
503 ShenandoahHeap* _heap;
504 ShenandoahMarkingContext* const _mark_context;
505
506 template <class T>
507 inline void do_oop_work(T* p) {
508 ShenandoahConcurrentMark::mark_through_ref<T, NONE, NO_DEDUP>(p, _heap, _queue, _mark_context);
509 }
510
511 public:
512 ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) :
513 _queue(q),
514 _heap(ShenandoahHeap::heap()),
515 _mark_context(_heap->marking_context()) {}
516
517 void do_oop(narrowOop* p) { do_oop_work(p); }
518 void do_oop(oop* p) { do_oop_work(p); }
519 };
520
521 class ShenandoahCMKeepAliveUpdateClosure : public OopClosure {
522 private:
523 ShenandoahObjToScanQueue* _queue;
524 ShenandoahHeap* _heap;
525 ShenandoahMarkingContext* const _mark_context;
526
527 template <class T>
528 inline void do_oop_work(T* p) {
529 ShenandoahConcurrentMark::mark_through_ref<T, SIMPLE, NO_DEDUP>(p, _heap, _queue, _mark_context);
530 }
531
532 public:
533 ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
534 _queue(q),
535 _heap(ShenandoahHeap::heap()),
536 _mark_context(_heap->marking_context()) {}
537
538 void do_oop(narrowOop* p) { do_oop_work(p); }
539 void do_oop(oop* p) { do_oop_work(p); }
540 };
541
542 class ShenandoahWeakUpdateClosure : public OopClosure {
543 private:
544 ShenandoahHeap* const _heap;
545
546 template <class T>
547 inline void do_oop_work(T* p) {
548 oop o = _heap->maybe_update_with_forwarded(p);
549 shenandoah_assert_marked_except(p, o, o == NULL);
550 }
551
552 public:
553 ShenandoahWeakUpdateClosure() : _heap(ShenandoahHeap::heap()) {}
554
555 void do_oop(narrowOop* p) { do_oop_work(p); }
556 void do_oop(oop* p) { do_oop_work(p); }
557 };
558
559 class ShenandoahWeakAssertNotForwardedClosure : public OopClosure {
560 private:
561 template <class T>
562 inline void do_oop_work(T* p) {
563 T o = RawAccess<>::oop_load(p);
564 if (!CompressedOops::is_null(o)) {
565 oop obj = CompressedOops::decode_not_null(o);
566 shenandoah_assert_not_forwarded(p, obj);
567 }
568 }
569
570 public:
571 ShenandoahWeakAssertNotForwardedClosure() {}
572
573 void do_oop(narrowOop* p) { do_oop_work(p); }
574 void do_oop(oop* p) { do_oop_work(p); }
575 };
576
577 class ShenandoahRefProcTaskProxy : public AbstractGangTask {
578 private:
579 AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
580 ShenandoahTaskTerminator* _terminator;
581
582 public:
583 ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
584 ShenandoahTaskTerminator* t) :
585 AbstractGangTask("Process reference objects in parallel"),
586 _proc_task(proc_task),
587 _terminator(t) {
588 }
589
590 void work(uint worker_id) {
591 ResourceMark rm;
592 HandleMark hm;
593 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
594 ShenandoahHeap* heap = ShenandoahHeap::heap();
595 ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator);
596 if (heap->has_forwarded_objects()) {
597 ShenandoahForwardedIsAliveClosure is_alive;
598 ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
599 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
600 } else {
601 ShenandoahIsAliveClosure is_alive;
602 ShenandoahCMKeepAliveClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
603 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
604 }
605 }
606 };
607
608 class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
609 private:
610 WorkGang* _workers;
611
612 public:
613 ShenandoahRefProcTaskExecutor(WorkGang* workers) :
614 _workers(workers) {
615 }
616
617 // Executes a task using worker threads.
618 void execute(ProcessTask& task, uint ergo_workers) {
619 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
620
621 ShenandoahHeap* heap = ShenandoahHeap::heap();
622 ShenandoahConcurrentMark* cm = heap->concurrent_mark();
623 ShenandoahPushWorkerQueuesScope scope(_workers, cm->task_queues(),
624 ergo_workers,
625 /* do_check = */ false);
626 uint nworkers = _workers->active_workers();
627 cm->task_queues()->reserve(nworkers);
628 ShenandoahTaskTerminator terminator(nworkers, cm->task_queues());
629 ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
630 _workers->run_task(&proc_task_proxy);
631 }
632 };
633
634 void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) {
635 assert(_heap->process_references(), "sanity");
636
637 ShenandoahPhaseTimings::Phase phase_root =
638 full_gc ?
639 ShenandoahPhaseTimings::full_gc_weakrefs :
640 ShenandoahPhaseTimings::weakrefs;
641
642 ShenandoahGCPhase phase(phase_root);
643
644 ReferenceProcessor* rp = _heap->ref_processor();
645
646 // NOTE: We cannot shortcut on has_discovered_references() here, because
647 // we will miss marking JNI Weak refs then, see implementation in
648 // ReferenceProcessor::process_discovered_references.
649 weak_refs_work_doit(full_gc);
650
651 rp->verify_no_references_recorded();
652 assert(!rp->discovery_enabled(), "Post condition");
653
654 }
655
656 void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) {
657 ReferenceProcessor* rp = _heap->ref_processor();
658
659 ShenandoahPhaseTimings::Phase phase_process =
660 full_gc ?
661 ShenandoahPhaseTimings::full_gc_weakrefs_process :
662 ShenandoahPhaseTimings::weakrefs_process;
663
664 ShenandoahPhaseTimings::Phase phase_process_termination =
665 full_gc ?
666 ShenandoahPhaseTimings::full_gc_weakrefs_termination :
667 ShenandoahPhaseTimings::weakrefs_termination;
668
669 shenandoah_assert_rp_isalive_not_installed();
670 ShenandoahIsAliveSelector is_alive;
671 ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
672
673 WorkGang* workers = _heap->workers();
674 uint nworkers = workers->active_workers();
675
676 rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
677 rp->set_active_mt_degree(nworkers);
678
679 assert(task_queues()->is_empty(), "Should be empty");
680
681 // complete_gc and keep_alive closures instantiated here are only needed for
682 // single-threaded path in RP. They share the queue 0 for tracking work, which
683 // simplifies implementation. Since RP may decide to call complete_gc several
684 // times, we need to be able to reuse the terminator.
685 uint serial_worker_id = 0;
686 ShenandoahTaskTerminator terminator(1, task_queues());
687 ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
688
689 ShenandoahRefProcTaskExecutor executor(workers);
690
691 ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
692
693 {
694 ShenandoahGCPhase phase(phase_process);
695 ShenandoahTerminationTracker phase_term(phase_process_termination);
696
697 // Process leftover weak oops: update them, if needed (using parallel version),
698 // or assert they do not need updating (using serial version) otherwise.
699 // Weak processor API requires us to visit the oops, even if we are not doing
700 // anything to them.
701 if (_heap->has_forwarded_objects()) {
702 ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id));
703 rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
704 &complete_gc, &executor,
705 &pt);
706
707 ShenandoahWeakUpdateClosure cl;
708 WeakProcessor::weak_oops_do(workers, is_alive.is_alive_closure(), &cl, 1);
709 } else {
710 ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id));
711 rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
712 &complete_gc, &executor,
713 &pt);
714
715 ShenandoahWeakAssertNotForwardedClosure cl;
716 WeakProcessor::weak_oops_do(is_alive.is_alive_closure(), &cl);
717 }
718
719 pt.print_all_references();
720
721 assert(task_queues()->is_empty(), "Should be empty");
722 }
723 }
724
725 class ShenandoahCancelledGCYieldClosure : public YieldClosure {
726 private:
727 ShenandoahHeap* const _heap;
728 public:
729 ShenandoahCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
730 virtual bool should_return() { return _heap->cancelled_gc(); }
731 };
732
733 class ShenandoahPrecleanCompleteGCClosure : public VoidClosure {
734 public:
735 void do_void() {
736 ShenandoahHeap* sh = ShenandoahHeap::heap();
737 ShenandoahConcurrentMark* scm = sh->concurrent_mark();
738 assert(sh->process_references(), "why else would we be here?");
739 ShenandoahTaskTerminator terminator(1, scm->task_queues());
740
741 ReferenceProcessor* rp = sh->ref_processor();
742 shenandoah_assert_rp_isalive_installed();
743
744 scm->mark_loop(0, &terminator, rp,
745 false, // not cancellable
746 false); // do not do strdedup
747 }
748 };
749
750 class ShenandoahPrecleanKeepAliveUpdateClosure : public OopClosure {
751 private:
752 ShenandoahObjToScanQueue* _queue;
753 ShenandoahHeap* _heap;
754 ShenandoahMarkingContext* const _mark_context;
755
756 template <class T>
757 inline void do_oop_work(T* p) {
758 ShenandoahConcurrentMark::mark_through_ref<T, CONCURRENT, NO_DEDUP>(p, _heap, _queue, _mark_context);
759 }
760
761 public:
762 ShenandoahPrecleanKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
763 _queue(q),
764 _heap(ShenandoahHeap::heap()),
765 _mark_context(_heap->marking_context()) {}
766
767 void do_oop(narrowOop* p) { do_oop_work(p); }
768 void do_oop(oop* p) { do_oop_work(p); }
769 };
770
771 class ShenandoahPrecleanTask : public AbstractGangTask {
772 private:
773 ReferenceProcessor* _rp;
774
775 public:
776 ShenandoahPrecleanTask(ReferenceProcessor* rp) :
777 AbstractGangTask("Precleaning task"),
778 _rp(rp) {}
779
780 void work(uint worker_id) {
781 assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
782 ShenandoahParallelWorkerSession worker_session(worker_id);
783
784 ShenandoahHeap* sh = ShenandoahHeap::heap();
785
786 ShenandoahObjToScanQueue* q = sh->concurrent_mark()->get_queue(worker_id);
787
788 ShenandoahCancelledGCYieldClosure yield;
789 ShenandoahPrecleanCompleteGCClosure complete_gc;
790
791 if (sh->has_forwarded_objects()) {
792 ShenandoahForwardedIsAliveClosure is_alive;
793 ShenandoahPrecleanKeepAliveUpdateClosure keep_alive(q);
794 ResourceMark rm;
795 _rp->preclean_discovered_references(&is_alive, &keep_alive,
796 &complete_gc, &yield,
797 NULL);
798 } else {
799 ShenandoahIsAliveClosure is_alive;
800 ShenandoahCMKeepAliveClosure keep_alive(q);
801 ResourceMark rm;
802 _rp->preclean_discovered_references(&is_alive, &keep_alive,
803 &complete_gc, &yield,
804 NULL);
805 }
806 }
807 };
808
809 void ShenandoahConcurrentMark::preclean_weak_refs() {
810 // Pre-cleaning weak references before diving into STW makes sense at the
811 // end of concurrent mark. This will filter out the references which referents
812 // are alive. Note that ReferenceProcessor already filters out these on reference
813 // discovery, and the bulk of work is done here. This phase processes leftovers
814 // that missed the initial filtering, i.e. when referent was marked alive after
815 // reference was discovered by RP.
816
817 assert(_heap->process_references(), "sanity");
818
819 // Shortcut if no references were discovered to avoid winding up threads.
820 ReferenceProcessor* rp = _heap->ref_processor();
821 if (!rp->has_discovered_references()) {
822 return;
823 }
824
825 assert(task_queues()->is_empty(), "Should be empty");
826
827 ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
828
829 shenandoah_assert_rp_isalive_not_installed();
830 ShenandoahIsAliveSelector is_alive;
831 ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
832
833 // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
834 // queues and other goodies. When upstream ReferenceProcessor starts supporting
835 // parallel precleans, we can extend this to more threads.
836 WorkGang* workers = _heap->workers();
837 uint nworkers = workers->active_workers();
838 assert(nworkers == 1, "This code uses only a single worker");
839 task_queues()->reserve(nworkers);
840
841 ShenandoahPrecleanTask task(rp);
842 workers->run_task(&task);
843
844 assert(task_queues()->is_empty(), "Should be empty");
845 }
846
847 void ShenandoahConcurrentMark::cancel() {
848 // Clean up marking stacks.
849 ShenandoahObjToScanQueueSet* queues = task_queues();
850 queues->clear();
851
852 // Cancel SATB buffers.
853 ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
854 }
855
856 ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
857 assert(task_queues()->get_reserved() > worker_id, "No reserved queue for worker id: %d", worker_id);
858 return _task_queues->queue(worker_id);
859 }
860
861 template <bool CANCELLABLE>
862 void ShenandoahConcurrentMark::mark_loop_prework(uint w, ShenandoahTaskTerminator *t, ReferenceProcessor *rp,
863 bool strdedup) {
864 ShenandoahObjToScanQueue* q = get_queue(w);
865
866 jushort* ld = _heap->get_liveness_cache(w);
867
868 // TODO: We can clean up this if we figure out how to do templated oop closures that
869 // play nice with specialized_oop_iterators.
870 if (_heap->unload_classes()) {
871 if (_heap->has_forwarded_objects()) {
872 if (strdedup) {
873 ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp);
874 mark_loop_work<ShenandoahMarkUpdateRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
875 } else {
876 ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp);
877 mark_loop_work<ShenandoahMarkUpdateRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
878 }
879 } else {
880 if (strdedup) {
881 ShenandoahMarkRefsMetadataDedupClosure cl(q, rp);
882 mark_loop_work<ShenandoahMarkRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
883 } else {
884 ShenandoahMarkRefsMetadataClosure cl(q, rp);
885 mark_loop_work<ShenandoahMarkRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
886 }
887 }
888 } else {
889 if (_heap->has_forwarded_objects()) {
890 if (strdedup) {
891 ShenandoahMarkUpdateRefsDedupClosure cl(q, rp);
892 mark_loop_work<ShenandoahMarkUpdateRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
893 } else {
894 ShenandoahMarkUpdateRefsClosure cl(q, rp);
895 mark_loop_work<ShenandoahMarkUpdateRefsClosure, CANCELLABLE>(&cl, ld, w, t);
896 }
897 } else {
898 if (strdedup) {
899 ShenandoahMarkRefsDedupClosure cl(q, rp);
900 mark_loop_work<ShenandoahMarkRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
901 } else {
902 ShenandoahMarkRefsClosure cl(q, rp);
903 mark_loop_work<ShenandoahMarkRefsClosure, CANCELLABLE>(&cl, ld, w, t);
904 }
905 }
906 }
907
908 _heap->flush_liveness_cache(w);
909 }
910
911 template <class T, bool CANCELLABLE>
912 void ShenandoahConcurrentMark::mark_loop_work(T* cl, jushort* live_data, uint worker_id, ShenandoahTaskTerminator *terminator) {
913 uintx stride = ShenandoahMarkLoopStride;
914
915 ShenandoahHeap* heap = ShenandoahHeap::heap();
916 ShenandoahObjToScanQueueSet* queues = task_queues();
917 ShenandoahObjToScanQueue* q;
918 ShenandoahMarkTask t;
919
920 /*
921 * Process outstanding queues, if any.
922 *
923 * There can be more queues than workers. To deal with the imbalance, we claim
924 * extra queues first. Since marking can push new tasks into the queue associated
925 * with this worker id, we come back to process this queue in the normal loop.
926 */
927 assert(queues->get_reserved() == heap->workers()->active_workers(),
928 "Need to reserve proper number of queues: reserved: %u, active: %u", queues->get_reserved(), heap->workers()->active_workers());
929
930 q = queues->claim_next();
931 while (q != NULL) {
932 if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
933 return;
934 }
935
936 for (uint i = 0; i < stride; i++) {
937 if (q->pop(t)) {
938 do_task<T>(q, cl, live_data, &t);
939 } else {
940 assert(q->is_empty(), "Must be empty");
941 q = queues->claim_next();
942 break;
943 }
944 }
945 }
946 q = get_queue(worker_id);
947
948 ShenandoahSATBBufferClosure drain_satb(q);
949 SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
950
951 /*
952 * Normal marking loop:
953 */
954 while (true) {
955 if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
956 return;
957 }
958
959 while (satb_mq_set.completed_buffers_num() > 0) {
960 satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
961 }
962
963 uint work = 0;
964 for (uint i = 0; i < stride; i++) {
965 if (q->pop(t) ||
966 queues->steal(worker_id, t)) {
967 do_task<T>(q, cl, live_data, &t);
968 work++;
969 } else {
970 break;
971 }
972 }
973
974 if (work == 0) {
975 // No work encountered in current stride, try to terminate.
976 // Need to leave the STS here otherwise it might block safepoints.
977 ShenandoahSuspendibleThreadSetLeaver stsl(CANCELLABLE && ShenandoahSuspendibleWorkers);
978 ShenandoahTerminationTimingsTracker term_tracker(worker_id);
979 ShenandoahTerminatorTerminator tt(heap);
980 if (terminator->offer_termination(&tt)) return;
981 }
982 }
983 }
984
985 bool ShenandoahConcurrentMark::claim_codecache() {
986 assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise");
987 return _claimed_codecache.try_set();
988 }
989
990 void ShenandoahConcurrentMark::clear_claim_codecache() {
991 assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise");
992 _claimed_codecache.unset();
993 }