1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "classfile/stringTable.hpp"
25 #include "gc/shared/gcTimer.hpp"
26 #include "gc/shared/isGCActiveMark.hpp"
27 #include "gc/shared/parallelCleaning.hpp"
28 #include "gc/shared/strongRootsScope.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
30 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
31 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
34 #include "gc/shenandoah/shenandoah_specialized_oop_closures.hpp"
35 #include "gc/shenandoah/brooksPointer.hpp"
36 #include "gc/shared/referenceProcessor.hpp"
37 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
38 #include "code/codeCache.hpp"
39 #include "classfile/symbolTable.hpp"
40 #include "classfile/systemDictionary.hpp"
41 #include "memory/iterator.inline.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "gc/shared/taskqueue.inline.hpp"
44
45 class ShenandoahInitMarkRootsClosure : public OopClosure {
46 SCMObjToScanQueue* _queue;
47 ShenandoahHeap* _heap;
48
49 public:
50 ShenandoahInitMarkRootsClosure(SCMObjToScanQueue* q) :
51 _queue(q),
52 _heap((ShenandoahHeap*) Universe::heap())
53 {
54 }
55
56 private:
57 template <class T>
58 inline void do_oop_work(T* p) {
59 T o = oopDesc::load_heap_oop(p);
60 if (! oopDesc::is_null(o)) {
61 oop obj = oopDesc::decode_heap_oop_not_null(o);
62 obj = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
63 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
64 "expect forwarded oop");
65 ShenandoahConcurrentMark::mark_and_push(obj, _heap, _queue);
66 }
67 }
68
69 public:
70 void do_oop(narrowOop* p) {
71 do_oop_work(p);
72 }
73
74 inline void do_oop(oop* p) {
75 do_oop_work(p);
76 }
77
78 };
79
80 class SCMUpdateRefsClosure: public OopClosure {
81 private:
82 ShenandoahHeap* _heap;
83 public:
84
85 SCMUpdateRefsClosure() : _heap(ShenandoahHeap::heap()) {
86 }
87
88 private:
89 template <class T>
90 inline void do_oop_work(T* p) {
91 T o = oopDesc::load_heap_oop(p);
92 if (! oopDesc::is_null(o)) {
93 oop obj = oopDesc::decode_heap_oop_not_null(o);
94 _heap->update_oop_ref_not_null(p, obj);
95 }
96 }
97
98 public:
99 inline void do_oop(oop* p) {
100 do_oop_work(p);
101 }
102
103 void do_oop(narrowOop* p) {
104 do_oop_work(p);
105 }
106 };
107
108 // Mark the object and add it to the queue to be scanned
109 template <class T, bool CL>
110 ShenandoahMarkObjsClosure<T, CL>::ShenandoahMarkObjsClosure(SCMObjToScanQueue* q, ReferenceProcessor* rp) :
111 _heap((ShenandoahHeap*)(Universe::heap())),
112 _queue(q),
113 _mark_refs(T(q, rp)),
114 _last_region_idx(0),
115 _live_data(0)
116 {
117 }
118
119 template <class T, bool CL>
120 ShenandoahMarkObjsClosure<T, CL>::~ShenandoahMarkObjsClosure() {
121 if (CL) {
122 ShenandoahHeapRegion *r = _heap->regions()->get(_last_region_idx);
123 r->increase_live_data(_live_data);
124 }
125 }
126
127 ShenandoahMarkUpdateRefsClosure::ShenandoahMarkUpdateRefsClosure(SCMObjToScanQueue* q, ReferenceProcessor* rp) :
128 MetadataAwareOopClosure(rp),
129 _queue(q),
130 _heap((ShenandoahHeap*) Universe::heap())
131 {
132 }
133
134 ShenandoahMarkRefsClosure::ShenandoahMarkRefsClosure(SCMObjToScanQueue* q, ReferenceProcessor* rp) :
135 MetadataAwareOopClosure(rp),
136 _queue(q),
137 _heap((ShenandoahHeap*) Universe::heap())
138 {
139 }
140
141 class ShenandoahInitMarkRootsTask : public AbstractGangTask {
142 private:
143 ShenandoahRootProcessor* _rp;
144 bool _process_refs;
145 public:
146 ShenandoahInitMarkRootsTask(ShenandoahRootProcessor* rp, bool process_refs) :
147 AbstractGangTask("Shenandoah init mark roots task"),
148 _rp(rp),
149 _process_refs(process_refs) {
150 }
151
152 void work(uint worker_id) {
153 ShenandoahHeap* heap = ShenandoahHeap::heap();
154 SCMObjToScanQueue* q = heap->concurrentMark()->get_queue(worker_id);
155 ShenandoahInitMarkRootsClosure mark_cl(q);
156 CLDToOopClosure cldCl(&mark_cl);
157 MarkingCodeBlobClosure blobsCl(&mark_cl, ! CodeBlobToOopClosure::FixRelocations);
158
159 ResourceMark m;
160 if (heap->concurrentMark()->unload_classes()) {
161 _rp->process_strong_roots(&mark_cl, _process_refs ? NULL : &mark_cl, &cldCl, &blobsCl, worker_id);
162 } else {
163 _rp->process_all_roots(&mark_cl, _process_refs ? NULL : &mark_cl, &cldCl, ShenandoahConcurrentCodeRoots ? NULL : &blobsCl, worker_id);
164 }
165 }
166 };
167
168 class ShenandoahUpdateRootsTask : public AbstractGangTask {
169 private:
170 ShenandoahRootProcessor* _rp;
171 public:
172 ShenandoahUpdateRootsTask(ShenandoahRootProcessor* rp) :
173 AbstractGangTask("Shenandoah update roots task"),
174 _rp(rp) {
175 }
176
177 void work(uint worker_id) {
178 ShenandoahHeap* heap = ShenandoahHeap::heap();
179 SCMUpdateRefsClosure cl;
180 CLDToOopClosure cldCl(&cl);
181
182 _rp->process_all_roots(&cl, &cl, &cldCl, NULL, worker_id);
183 }
184 };
185
186 class SCMConcurrentMarkingTask : public AbstractGangTask {
187 private:
188 ShenandoahConcurrentMark* _cm;
189 ParallelTaskTerminator* _terminator;
190 bool _update_refs;
191
192 public:
193 SCMConcurrentMarkingTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator, bool update_refs) :
194 AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator), _update_refs(update_refs) {
195 }
196
197
198 void work(uint worker_id) {
199
200 SCMObjToScanQueue* q = _cm->get_queue(worker_id);
201 ReferenceProcessor* rp;
202 if (_cm->process_references()) {
203 rp = ShenandoahHeap::heap()->ref_processor();
204 } else {
205 rp = NULL;
206 }
207 if (ShenandoahConcurrentCodeRoots && _cm->claim_codecache()) {
208 if (! _cm->unload_classes()) {
209 ShenandoahMarkRefsClosure cl(q, rp);
210 CodeBlobToOopClosure blobs(&cl, ! CodeBlobToOopClosure::FixRelocations);
211 MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
212 CodeCache::blobs_do(&blobs);
213 }
214 }
215 if (_update_refs) {
216 ShenandoahMarkObjsClosure<ShenandoahMarkUpdateRefsClosure, true> cl(q, rp);
217 _cm->concurrent_mark_loop(&cl, worker_id, q, _terminator);
218 } else {
219 ShenandoahMarkObjsClosure<ShenandoahMarkRefsClosure, true> cl(q, rp);
220 _cm->concurrent_mark_loop(&cl, worker_id, q, _terminator);
221 }
222 ShenandoahHeap* heap = ShenandoahHeap::heap();
223 }
224 };
225
226 class SCMFinalMarkingTask : public AbstractGangTask {
227 private:
228 ShenandoahConcurrentMark* _cm;
229 ParallelTaskTerminator* _terminator;
230 bool _update_refs;
231 bool _count_live;
232
233 public:
234 SCMFinalMarkingTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator, bool update_refs, bool count_live) :
235 AbstractGangTask("Shenandoah Final Marking"), _cm(cm), _terminator(terminator), _update_refs(update_refs), _count_live(count_live) {
236 }
237
238 void work(uint worker_id) {
239 ReferenceProcessor* rp;
240 if (_cm->process_references()) {
241 rp = ShenandoahHeap::heap()->ref_processor();
242 } else {
243 rp = NULL;
244 }
245 SCMObjToScanQueue* q = _cm->get_queue(worker_id);
246
247 // Templates need constexprs, so we have to switch by the flags ourselves.
248 if (_update_refs) {
249 if (_count_live) {
250 ShenandoahMarkObjsClosure<ShenandoahMarkUpdateRefsClosure, true> cl(q, rp);
251 _cm->final_mark_loop(&cl, worker_id, q, _terminator);
252 } else {
253 ShenandoahMarkObjsClosure<ShenandoahMarkUpdateRefsClosure, false> cl(q, rp);
254 _cm->final_mark_loop(&cl, worker_id, q, _terminator);
255 }
256 } else {
257 if (_count_live) {
258 ShenandoahMarkObjsClosure<ShenandoahMarkRefsClosure, true> cl(q, rp);
259 _cm->final_mark_loop(&cl, worker_id, q, _terminator);
260 } else {
261 ShenandoahMarkObjsClosure<ShenandoahMarkRefsClosure, false> cl(q, rp);
262 _cm->final_mark_loop(&cl, worker_id, q, _terminator);
263 }
264 }
265 }
266 };
267
268 void ShenandoahConcurrentMark::mark_roots() {
269 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
270
271 ShenandoahHeap* heap = ShenandoahHeap::heap();
272
273 ClassLoaderDataGraph::clear_claimed_marks();
274
275 ShenandoahRootProcessor root_proc(heap, _max_conc_worker_id, ShenandoahCollectorPolicy::scan_thread_roots);
276 TASKQUEUE_STATS_ONLY(reset_taskqueue_stats());
277 ShenandoahInitMarkRootsTask mark_roots(&root_proc, process_references());
278 heap->conc_workers()->run_task(&mark_roots);
279 if (ShenandoahConcurrentCodeRoots) {
280 clear_claim_codecache();
281 }
282 }
283
284 void ShenandoahConcurrentMark::init_mark_roots() {
285 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
286
287 ShenandoahHeap* heap = ShenandoahHeap::heap();
288
289 // Set up ref processing and class unloading.
290 ShenandoahCollectorPolicy* policy = heap->shenandoahPolicy();
291 set_process_references(policy->process_references());
292 set_unload_classes(policy->unload_classes());
293
294 mark_roots();
295 }
296
297 void ShenandoahConcurrentMark::update_roots() {
298 ShenandoahHeap* heap = ShenandoahHeap::heap();
299
300 ClassLoaderDataGraph::clear_claimed_marks();
301
302 ShenandoahRootProcessor root_proc(heap, heap->max_parallel_workers(), ShenandoahCollectorPolicy::update_thread_roots);
303 ShenandoahUpdateRootsTask update_roots(&root_proc);
304 heap->workers()->run_task(&update_roots);
305
306 }
307
308 void ShenandoahConcurrentMark::final_update_roots() {
309 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
310
311 COMPILER2_PRESENT(DerivedPointerTable::clear());
312
313 update_roots();
314
315 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
316 }
317
318
319 void ShenandoahConcurrentMark::initialize() {
320 _max_conc_worker_id = MAX2((uint) ConcGCThreads, 1U);
321 _task_queues = new SCMObjToScanQueueSet((int) _max_conc_worker_id);
322
323 for (uint i = 0; i < _max_conc_worker_id; ++i) {
324 SCMObjToScanQueue* task_queue = new SCMObjToScanQueue();
325 task_queue->initialize();
326 _task_queues->register_queue(i, task_queue);
327 }
328 _process_references = false;
329 _unload_classes = false;
330 _claimed_codecache = 0;
331
332 JavaThread::satb_mark_queue_set().set_buffer_size(1014 /* G1SATBBufferSize */);
333 }
334
335 void ShenandoahConcurrentMark::mark_from_roots() {
336
337 ShenandoahHeap* sh = (ShenandoahHeap *) Universe::heap();
338
339 bool update_refs = sh->need_update_refs();
340
341 sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::conc_mark);
342
343 if (process_references()) {
344 ReferenceProcessor* rp = sh->ref_processor();
345 // enable ("weak") refs discovery
346 rp->enable_discovery(true /*verify_no_refs*/);
347 rp->setup_policy(sh->is_full_gc_in_progress()); // snapshot the soft ref policy to be used in this cycle
348 }
349
350 if (UseShenandoahOWST) {
351 ShenandoahTaskTerminator terminator(_max_conc_worker_id, _task_queues);
352 SCMConcurrentMarkingTask markingTask = SCMConcurrentMarkingTask(this, &terminator, update_refs);
353 sh->conc_workers()->run_task(&markingTask);
354 } else {
355 ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues);
356 SCMConcurrentMarkingTask markingTask = SCMConcurrentMarkingTask(this, &terminator, update_refs);
357 sh->conc_workers()->run_task(&markingTask);
358 }
359
360 if (! sh->cancelled_concgc()) {
361 TASKQUEUE_STATS_ONLY(print_taskqueue_stats());
362 }
363 TASKQUEUE_STATS_ONLY(reset_taskqueue_stats());
364
365 sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::conc_mark);
366 }
367
368 class FinishDrainSATBBuffersTask : public AbstractGangTask {
369 private:
370 ShenandoahConcurrentMark* _cm;
371 ParallelTaskTerminator* _terminator;
372 public:
373 FinishDrainSATBBuffersTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator) :
374 AbstractGangTask("Finish draining SATB buffers"), _cm(cm), _terminator(terminator) {
375 }
376
377 void work(uint worker_id) {
378 _cm->drain_satb_buffers(worker_id, true);
379 }
380 };
381
382 void ShenandoahConcurrentMark::finish_mark_from_roots() {
383
384 IsGCActiveMark is_active;
385
386 ShenandoahHeap* sh = (ShenandoahHeap *) Universe::heap();
387
388 TASKQUEUE_STATS_ONLY(reset_taskqueue_stats());
389
390 sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::drain_satb);
391 {
392 StrongRootsScope scope(_max_conc_worker_id);
393 if (UseShenandoahOWST) {
394 ShenandoahTaskTerminator terminator(_max_conc_worker_id, _task_queues);
395 // drain_satb_buffers(0, true);
396 FinishDrainSATBBuffersTask drain_satb_buffers(this, &terminator);
397 sh->conc_workers()->run_task(&drain_satb_buffers);
398 } else {
399 ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues);
400 // drain_satb_buffers(0, true);
401 FinishDrainSATBBuffersTask drain_satb_buffers(this, &terminator);
402 sh->conc_workers()->run_task(&drain_satb_buffers);
403 }
404 sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::drain_satb);
405 }
406
407 shared_finish_mark_from_roots(/* full_gc = */ false);
408
409 sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::update_roots);
410 if (sh->need_update_refs()) {
411 final_update_roots();
412 }
413 sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::update_roots);
414
415 TASKQUEUE_STATS_ONLY(print_taskqueue_stats());
416
417 #ifdef ASSERT
418 verify_roots();
419
420 if (ShenandoahDumpHeapAfterConcurrentMark) {
421 sh->ensure_parsability(false);
422 sh->print_all_refs("post-mark");
423 }
424 #endif
425 }
426
427 void ShenandoahConcurrentMark::shared_finish_mark_from_roots(bool full_gc) {
428
429 ShenandoahHeap* sh = ShenandoahHeap::heap();
430 ShenandoahCollectorPolicy* policy = sh->shenandoahPolicy();
431
432 // Finally mark everything else we've got in our queues during the previous steps.
433 {
434 policy->record_phase_start(full_gc ?
435 ShenandoahCollectorPolicy::full_gc_mark_drain_queues :
436 ShenandoahCollectorPolicy::drain_queues);
437 bool count_live = !(ShenandoahNoLivenessFullGC && full_gc); // we do not need liveness data for full GC
438 if (UseShenandoahOWST) {
439 ShenandoahTaskTerminator terminator(_max_conc_worker_id, _task_queues);
440 SCMFinalMarkingTask markingTask = SCMFinalMarkingTask(this, &terminator, sh->need_update_refs(), count_live);
441 sh->conc_workers()->run_task(&markingTask);
442 } else {
443 ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues);
444 SCMFinalMarkingTask markingTask = SCMFinalMarkingTask(this, &terminator, sh->need_update_refs(), count_live);
445 sh->conc_workers()->run_task(&markingTask);
446 }
447 policy->record_phase_end(full_gc ?
448 ShenandoahCollectorPolicy::full_gc_mark_drain_queues :
449 ShenandoahCollectorPolicy::drain_queues);
450 }
451
452 #ifdef ASSERT
453 for (int i = 0; i < (int) _max_conc_worker_id; i++) {
454 assert(_task_queues->queue(i)->is_empty(), "Should be empty");
455 }
456 #endif
457
458 // When we're done marking everything, we process weak references.
459 policy->record_phase_start(full_gc ?
460 ShenandoahCollectorPolicy::full_gc_mark_weakrefs :
461 ShenandoahCollectorPolicy::weakrefs);
462 if (process_references()) {
463 weak_refs_work();
464 }
465 policy->record_phase_end(full_gc ?
466 ShenandoahCollectorPolicy::full_gc_mark_weakrefs :
467 ShenandoahCollectorPolicy::weakrefs);
468
469 // And finally finish class unloading
470 policy->record_phase_start(full_gc ?
471 ShenandoahCollectorPolicy::full_gc_mark_class_unloading :
472 ShenandoahCollectorPolicy::class_unloading);
473 if (unload_classes()) {
474 ShenandoahForwardedIsAliveClosure is_alive;
475 // Unload classes and purge SystemDictionary.
476 bool purged_class = SystemDictionary::do_unloading(&is_alive, false);
477 ParallelCleaningTask unlink_task(&is_alive, true, true, max_conc_worker_id(), purged_class);
478 sh->conc_workers()->run_task(&unlink_task);
479 ClassLoaderDataGraph::purge();
480 }
481 policy->record_phase_end(full_gc ?
482 ShenandoahCollectorPolicy::full_gc_mark_class_unloading :
483 ShenandoahCollectorPolicy::class_unloading);
484
485 #ifdef ASSERT
486 for (int i = 0; i < (int) _max_conc_worker_id; i++) {
487 assert(_task_queues->queue(i)->is_empty(), "Should be empty");
488 }
489 #endif
490
491 }
492
493 #ifdef ASSERT
494 template <class T>
495 void ShenandoahVerifyRootsClosure1::do_oop_work(T* p) {
496 T o = oopDesc::load_heap_oop(p);
497 if (! oopDesc::is_null(o)) {
498 oop obj = oopDesc::decode_heap_oop_not_null(o);
499 if (! oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj))) {
500 tty->print_cr("from-space marked: %s, to-space marked: %s, unload_classes: %s", BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_current(obj)), BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_current(ShenandoahBarrierSet::resolve_oop_static_not_null(obj))), BOOL_TO_STR(ShenandoahHeap::heap()->concurrentMark()->unload_classes()));
501 }
502 guarantee(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "oop must not be forwarded");
503 guarantee(ShenandoahHeap::heap()->is_marked_current(obj), "oop must be marked");
504 }
505 }
506
507 void ShenandoahVerifyRootsClosure1::do_oop(oop* p) {
508 do_oop_work(p);
509 }
510
511 void ShenandoahVerifyRootsClosure1::do_oop(narrowOop* p) {
512 do_oop_work(p);
513 }
514
515 void ShenandoahConcurrentMark::verify_roots() {
516 ShenandoahVerifyRootsClosure1 cl;
517 CodeBlobToOopClosure blobsCl(&cl, false);
518 CLDToOopClosure cldCl(&cl);
519 ClassLoaderDataGraph::clear_claimed_marks();
520 ShenandoahRootProcessor rp(ShenandoahHeap::heap(), 1);
521 rp.process_all_roots(&cl, &cl, &cldCl, &blobsCl, 0);
522
523 }
524 #endif
525
526 class ShenandoahSATBThreadsClosure : public ThreadClosure {
527 ShenandoahSATBBufferClosure* _satb_cl;
528 int _thread_parity;
529
530 public:
531 ShenandoahSATBThreadsClosure(ShenandoahSATBBufferClosure* satb_cl) :
532 _satb_cl(satb_cl),
533 _thread_parity(Threads::thread_claim_parity()) {}
534
535 void do_thread(Thread* thread) {
536 if (thread->is_Java_thread()) {
537 if (thread->claim_oops_do(true, _thread_parity)) {
538 JavaThread* jt = (JavaThread*)thread;
539 jt->satb_mark_queue().apply_closure_and_empty(_satb_cl);
540 }
541 } else if (thread->is_VM_thread()) {
542 if (thread->claim_oops_do(true, _thread_parity)) {
543 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
544 }
545 }
546 }
547 };
548
549 void ShenandoahConcurrentMark::drain_satb_buffers(uint worker_id, bool remark) {
550 SCMObjToScanQueue* q = get_queue(worker_id);
551 ShenandoahSATBBufferClosure cl(q);
552
553 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
554 while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
555
556 if (remark) {
557 ShenandoahSATBThreadsClosure tc(&cl);
558 Threads::threads_do(&tc);
559 }
560 }
561
562 #if TASKQUEUE_STATS
563 void ShenandoahConcurrentMark::print_taskqueue_stats_hdr(outputStream* const st) {
564 st->print_raw_cr("GC Task Stats");
565 st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr();
566 st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr();
567 }
568
569 void ShenandoahConcurrentMark::print_taskqueue_stats() const {
570 if (!log_develop_is_enabled(Trace, gc, task, stats)) {
571 return;
572 }
573 Log(gc, task, stats) log;
574 ResourceMark rm;
575 outputStream* st = log.trace_stream();
576 print_taskqueue_stats_hdr(st);
577
578 ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
579 TaskQueueStats totals;
580 const int n = sh->max_conc_workers();
581 for (int i = 0; i < n; ++i) {
582 st->print(INT32_FORMAT_W(3), i);
583 _task_queues->queue(i)->stats.print(st);
584 st->cr();
585 totals += _task_queues->queue(i)->stats;
586 }
587 st->print("tot "); totals.print(st); st->cr();
588 DEBUG_ONLY(totals.verify());
589
590 }
591
592 void ShenandoahConcurrentMark::reset_taskqueue_stats() {
593 ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
594 const int n = sh->max_conc_workers();
595 for (int i = 0; i < n; ++i) {
596 _task_queues->queue(i)->stats.reset();
597 }
598 }
599 #endif // TASKQUEUE_STATS
600
601 // Weak Reference Closures
602 class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
603 uint _worker_id;
604 ParallelTaskTerminator* _terminator;
605
606 public:
607 ShenandoahCMDrainMarkingStackClosure(uint worker_id, ParallelTaskTerminator* t):
608 _worker_id(worker_id),
609 _terminator(t) {
610 }
611
612
613 void do_void() {
614
615 ShenandoahHeap* sh = ShenandoahHeap::heap();
616 ShenandoahConcurrentMark* scm = sh->concurrentMark();
617 ReferenceProcessor* rp;
618 if (scm->process_references()) {
619 rp = ShenandoahHeap::heap()->ref_processor();
620 } else {
621 rp = NULL;
622 }
623 SCMObjToScanQueue* q = scm->get_queue(_worker_id);
624 if (sh->need_update_refs()) {
625 ShenandoahMarkObjsClosure<ShenandoahMarkUpdateRefsClosure, true> cl(q, rp);
626 scm->final_mark_loop(&cl, _worker_id, q, _terminator);
627 } else {
628 ShenandoahMarkObjsClosure<ShenandoahMarkRefsClosure, true> cl(q, rp);
629 scm->final_mark_loop(&cl, _worker_id, q, _terminator);
630 }
631 }
632 };
633
634
635 class ShenandoahCMKeepAliveClosure: public OopClosure {
636 SCMObjToScanQueue* _queue;
637 ShenandoahHeap* _sh;
638
639 public:
640 ShenandoahCMKeepAliveClosure(SCMObjToScanQueue* q) :
641 _queue(q) {
642 _sh = (ShenandoahHeap*) Universe::heap();
643 }
644
645 private:
646 template <class T>
647 inline void do_oop_work(T* p) {
648
649 T o = oopDesc::load_heap_oop(p);
650 if (! oopDesc::is_null(o)) {
651 oop obj = oopDesc::decode_heap_oop_not_null(o);
652 assert(oopDesc::unsafe_equals(obj, oopDesc::bs()->read_barrier(obj)), "only get updated oops in weak ref processing");
653
654 #ifdef ASSERT
655 if (log_is_enabled(Trace, gc, ref)) {
656 ResourceMark rm;
657 outputStream* out = Log(gc, ref)::trace_stream();
658 out->print("\twe're looking at location "
659 "*"PTR_FORMAT" = "PTR_FORMAT,
660 p2i(p), p2i((void*) obj));
661 obj->print_on(out);
662 }
663 #endif
664 ShenandoahConcurrentMark::mark_and_push(obj, _sh, _queue);
665 }
666 }
667
668 public:
669 void do_oop(narrowOop* p) {
670 do_oop_work(p);
671 }
672
673
674 void do_oop(oop* p) {
675 do_oop_work(p);
676 }
677
678 };
679
680 class ShenandoahCMKeepAliveUpdateClosure: public OopClosure {
681 SCMObjToScanQueue* _queue;
682 ShenandoahHeap* _sh;
683
684 public:
685 ShenandoahCMKeepAliveUpdateClosure(SCMObjToScanQueue* q) :
686 _queue(q) {
687 _sh = (ShenandoahHeap*) Universe::heap();
688 }
689
690 private:
691 template <class T>
692 inline void do_oop_work(T* p) {
693 T o = oopDesc::load_heap_oop(p);
694 if (! oopDesc::is_null(o)) {
695 oop obj = oopDesc::decode_heap_oop_not_null(o);
696 obj = _sh->update_oop_ref_not_null(p, obj);
697 assert(oopDesc::unsafe_equals(obj, oopDesc::bs()->read_barrier(obj)), "only get updated oops in weak ref processing");
698 #ifdef ASSERT
699 if (log_is_enabled(Trace, gc, ref)) {
700 ResourceMark rm;
701 outputStream* out = Log(gc, ref)::trace_stream();
702 out->print("\twe're looking at location "
703 "*"PTR_FORMAT" = "PTR_FORMAT,
704 p2i(p), p2i((void*) obj));
705 obj->print_on(out);
706 }
707 #endif
708 ShenandoahConcurrentMark::mark_and_push(obj, _sh, _queue);
709 }
710 }
711
712 public:
713 void do_oop(narrowOop* p) {
714 do_oop_work(p);
715 }
716
717 void do_oop(oop* p) {
718 do_oop_work(p);
719 }
720
721 };
722
723 class ShenandoahRefProcTaskProxy : public AbstractGangTask {
724
725 private:
726 AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
727 ParallelTaskTerminator* _terminator;
728 public:
729
730 ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
731 ParallelTaskTerminator* t) :
732 AbstractGangTask("Process reference objects in parallel"),
733 _proc_task(proc_task),
734 _terminator(t) {
735 }
736
737 void work(uint worker_id) {
738 ShenandoahHeap* heap = ShenandoahHeap::heap();
739 ShenandoahForwardedIsAliveClosure is_alive;
740 ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator);
741 if (heap->need_update_refs()) {
742 ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrentMark()->get_queue(worker_id));
743 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
744 } else {
745 ShenandoahCMKeepAliveClosure keep_alive(heap->concurrentMark()->get_queue(worker_id));
746 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
747 }
748 }
749 };
750
751 class ShenandoahRefEnqueueTaskProxy : public AbstractGangTask {
752
753 private:
754 AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task;
755
756 public:
757
758 ShenandoahRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) :
759 AbstractGangTask("Enqueue reference objects in parallel"),
760 _enqueue_task(enqueue_task) {
761 }
762
763 void work(uint worker_id) {
764 _enqueue_task.work(worker_id);
765 }
766 };
767
768 class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
769
770 private:
771 WorkGang* _workers;
772
773 public:
774
775 ShenandoahRefProcTaskExecutor() : _workers(ShenandoahHeap::heap()->conc_workers()) {
776 }
777
778 // Executes a task using worker threads.
779 void execute(ProcessTask& task) {
780 ShenandoahConcurrentMark* cm = ShenandoahHeap::heap()->concurrentMark();
781 if (UseShenandoahOWST) {
782 ShenandoahTaskTerminator terminator(cm->max_conc_worker_id(), cm->task_queues());
783 ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
784 _workers->run_task(&proc_task_proxy);
785 } else {
786 ParallelTaskTerminator terminator(cm->max_conc_worker_id(), cm->task_queues());
787 ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
788 _workers->run_task(&proc_task_proxy);
789 }
790 }
791
792 void execute(EnqueueTask& task) {
793 ShenandoahRefEnqueueTaskProxy enqueue_task_proxy(task);
794 _workers->run_task(&enqueue_task_proxy);
795 }
796 };
797
798
799 void ShenandoahConcurrentMark::weak_refs_work() {
800 assert(process_references(), "sanity");
801 ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
802 ReferenceProcessor* rp = sh->ref_processor();
803
804 // Setup collector policy for softref cleaning.
805 bool clear_soft_refs = sh->collector_policy()->use_should_clear_all_soft_refs(true /* bogus arg*/);
806 log_develop_debug(gc, ref)("clearing soft refs: %s", BOOL_TO_STR(clear_soft_refs));
807 rp->setup_policy(clear_soft_refs);
808
809 uint serial_worker_id = 0;
810 ShenandoahForwardedIsAliveClosure is_alive;
811 ParallelTaskTerminator terminator(1, task_queues());
812 ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator);
813 ShenandoahRefProcTaskExecutor executor;
814
815 log_develop_trace(gc, ref)("start processing references");
816
817 if (sh->need_update_refs()) {
818 ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id));
819 rp->process_discovered_references(&is_alive, &keep_alive,
820 &complete_gc, &executor,
821 NULL);
822 } else {
823 ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id));
824 rp->process_discovered_references(&is_alive, &keep_alive,
825 &complete_gc, &executor,
826 NULL);
827 }
828
829 #ifdef ASSERT
830 for (int i = 0; i < (int) _max_conc_worker_id; i++) {
831 assert(_task_queues->queue(i)->is_empty(), "Should be empty");
832 }
833 #endif
834
835 log_develop_trace(gc, ref)("finished processing references");
836 log_develop_trace(gc, ref)("start enqueuing references");
837
838 rp->enqueue_discovered_references(&executor);
839
840 log_develop_trace(gc, ref)("finished enqueueing references");
841
842 rp->verify_no_references_recorded();
843 assert(!rp->discovery_enabled(), "Post condition");
844
845 }
846
847 void ShenandoahConcurrentMark::cancel() {
848 ShenandoahHeap* sh = ShenandoahHeap::heap();
849
850 // Cancel weak-ref discovery.
851 if (process_references()) {
852 ReferenceProcessor* rp = sh->ref_processor();
853 rp->abandon_partial_discovery();
854 rp->disable_discovery();
855 }
856
857 // Clean up marking stacks.
858 SCMObjToScanQueueSet* queues = task_queues();
859 for (uint i = 0; i < _max_conc_worker_id; ++i) {
860 clear_queue(queues->queue(i));
861 }
862
863 // Cancel SATB buffers.
864 JavaThread::satb_mark_queue_set().abandon_partial_marking();
865 }
866
867 SCMObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
868 assert (worker_id < _max_conc_worker_id, "valid worker id: %d", worker_id);
869 return _task_queues->queue(worker_id);
870 }
871
872 void ShenandoahConcurrentMark::clear_queue(SCMObjToScanQueue *q) {
873 q->set_empty();
874 q->overflow_stack()->clear();
875 q->clear_buffer();
876 }
877
878 template <class T, bool CL>
879 void ShenandoahConcurrentMark::concurrent_mark_loop(ShenandoahMarkObjsClosure<T, CL>* cl,
880 uint worker_id,
881 SCMObjToScanQueue* q,
882 ParallelTaskTerminator* terminator) {
883 ShenandoahHeap* heap = ShenandoahHeap::heap();
884 int seed = 17;
885 while (true) {
886 if (heap->cancelled_concgc()) clear_queue(q);
887 if (heap->cancelled_concgc() ||
888 (!try_queue(q, cl) &&
889 !try_draining_an_satb_buffer(q) &&
890 !try_to_steal(worker_id, cl, &seed))
891 ) {
892 if (terminator->offer_termination()) break;
893 }
894 }
895 }
896
897 template <class T, bool CL>
898 void ShenandoahConcurrentMark::final_mark_loop(ShenandoahMarkObjsClosure<T, CL>* cl,
899 uint worker_id,
900 SCMObjToScanQueue* q,
901 ParallelTaskTerminator* terminator) {
902 int seed = 17;
903 while (true) {
904 if (!try_queue(q, cl) &&
905 !try_to_steal(worker_id, cl, &seed)) {
906 if (terminator->offer_termination()) break;
907 }
908 }
909 }
910
911 void ShenandoahConcurrentMark::set_process_references(bool pr) {
912 _process_references = pr;
913 }
914
915 bool ShenandoahConcurrentMark::process_references() const {
916 return _process_references;
917 }
918
919 void ShenandoahConcurrentMark::set_unload_classes(bool uc) {
920 _unload_classes = uc;
921 }
922
923 bool ShenandoahConcurrentMark::unload_classes() const {
924 return _unload_classes;
925 }
926
927 bool ShenandoahConcurrentMark::claim_codecache() {
928 assert(ShenandoahConcurrentCodeRoots, "must not be called otherwise");
929 jbyte old = Atomic::cmpxchg(1, &_claimed_codecache, 0);
930 return old == 0;
931 }
932
933 void ShenandoahConcurrentMark::clear_claim_codecache() {
934 assert(ShenandoahConcurrentCodeRoots, "must not be called otherwise");
935 _claimed_codecache = 0;
936 }
937
938 // Generate Shenandoah specialized oop_oop_iterate functions.
939 SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_SHENANDOAH(ALL_KLASS_OOP_OOP_ITERATE_DEFN)