210 }
211
212 _heap->set_bytes_allocated_since_cm(0);
213
214 _heap->set_need_update_refs(false);
215
216 _heap->set_full_gc_in_progress(false);
217 }
218
219 _gc_timer->register_gc_end();
220
221 policy->record_full_gc();
222
223 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_heapdumps);
224 _heap->post_full_gc_dump(_gc_timer);
225 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_heapdumps);
226
227 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc);
228
229 oopDesc::set_bs(old_bs);
230 }
231
232 #ifdef ASSERT
233 class VerifyNotForwardedPointersClosure : public MetadataAwareOopClosure {
234 private:
235 template <class T>
236 inline void do_oop_work(T* p) {
237 T o = oopDesc::load_heap_oop(p);
238 if (! oopDesc::is_null(o)) {
239 oop obj = oopDesc::decode_heap_oop_not_null(o);
240 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
241 "expect forwarded oop");
242 ShenandoahHeap* heap = ShenandoahHeap::heap();
243 if (! heap->is_marked_complete(obj)) {
244 tty->print_cr("ref region humongous? %s", BOOL_TO_STR(heap->heap_region_containing(p)->is_humongous()));
245 }
246 assert(heap->is_marked_complete(obj), "must be marked");
247 assert(! heap->allocated_after_complete_mark_start((HeapWord*) obj), "must be truly marked");
248 }
249 }
288 ShenandoahConcurrentMark* cm = _heap->concurrentMark();
289
290 cm->set_process_references(true);
291 cm->set_unload_classes(true);
292
293 ReferenceProcessor* rp = _heap->ref_processor();
294 // enable ("weak") refs discovery
295 rp->enable_discovery(true /*verify_no_refs*/);
296 rp->setup_policy(true); // snapshot the soft ref policy to be used in this cycle
297 rp->set_active_mt_degree(_heap->workers()->active_workers());
298
299 COMPILER2_PRESENT(DerivedPointerTable::clear());
300 cm->update_roots();
301 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
302
303 cm->mark_roots();
304 cm->shared_finish_mark_from_roots(/* full_gc = */ true);
305
306 _heap->swap_mark_bitmaps();
307
308 if (UseShenandoahMatrix) {
309 if (PrintShenandoahMatrix) {
310 outputStream* log = Log(gc)::info_stream();
311 _heap->connection_matrix()->print_on(log);
312 }
313 if (VerifyShenandoahMatrix) {
314 _heap->verify_matrix();
315 }
316 }
317
318 if (VerifyDuringGC) {
319 HandleMark hm; // handle scope
320 // Universe::heap()->prepare_for_verify();
321 _heap->prepare_for_verify();
322 // Note: we can verify only the heap here. When an object is
323 // marked, the previous value of the mark word (including
324 // identity hash values, ages, etc) is preserved, and the mark
325 // word is set to markOop::marked_value - effectively removing
326 // any hash values from the mark word. These hash values are
327 // used when verifying the dictionaries and so removing them
328 // from the mark word can make verification of the dictionaries
329 // fail. At the end of the GC, the original mark word values
330 // (including hash values) are restored to the appropriate
331 // objects.
332 // Universe::heap()->verify(VerifySilently, VerifyOption_G1UseMarkWord);
333 _heap->verify(VerifyOption_G1UseMarkWord);
334 }
335
336 #ifdef ASSERT
337 ShenandoahMCVerifyAfterMarkingRegionClosure cl;
477 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
478 ShenandoahHeap* heap = ShenandoahHeap::heap();
479
480 ShenandoahMCReclaimHumongousRegionClosure cl;
481 heap->heap_region_iterate(&cl);
482
483 // Initialize copy queues.
484 for (uint i = 0; i < heap->max_workers(); i++) {
485 copy_queues[i] = new ShenandoahHeapRegionSet(heap->max_regions());
486 }
487
488 ShenandoahHeapRegionSet* from_regions = heap->regions();
489 from_regions->clear_current_index();
490 ShenandoahPrepareForCompactionTask prepare_task(from_regions, copy_queues);
491 heap->workers()->run_task(&prepare_task);
492 }
493
494 class ShenandoahAdjustPointersClosure : public MetadataAwareOopClosure {
495 private:
496 ShenandoahHeap* _heap;
497
498 public:
499
500 ShenandoahAdjustPointersClosure() : _heap(ShenandoahHeap::heap()) {
501 }
502
503 private:
504 template <class T>
505 inline void do_oop_work(T* p) {
506 T o = oopDesc::load_heap_oop(p);
507 if (! oopDesc::is_null(o)) {
508 oop obj = oopDesc::decode_heap_oop_not_null(o);
509 assert(_heap->is_marked_complete(obj), "must be marked");
510 oop forw = oop(BrooksPointer::get_raw(obj));
511 oopDesc::encode_store_heap_oop(p, forw);
512 }
513 }
514 public:
515 void do_oop(oop* p) {
516 do_oop_work(p);
517 }
518 void do_oop(narrowOop* p) {
519 do_oop_work(p);
520 }
521 };
522
523 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
524 private:
525 ShenandoahAdjustPointersClosure* _cl;
526 ShenandoahHeap* _heap;
527 public:
528 ShenandoahAdjustPointersObjectClosure(ShenandoahAdjustPointersClosure* cl) :
529 _cl(cl), _heap(ShenandoahHeap::heap()) {
530 }
531 void do_object(oop p) {
532 assert(_heap->is_marked_complete(p), "must be marked");
533 p->oop_iterate(_cl);
534 }
535 };
536
537 class ShenandoahAdjustPointersTask : public AbstractGangTask {
538 private:
539 ShenandoahHeapRegionSet* _regions;
540 public:
541
542 ShenandoahAdjustPointersTask(ShenandoahHeapRegionSet* regions) :
543 AbstractGangTask("Shenandoah Adjust Pointers Task"),
544 _regions(regions) {
545 }
546
547 void work(uint worker_id) {
548 ShenandoahHeap* heap = ShenandoahHeap::heap();
549 ShenandoahHeapRegion* r = _regions->claim_next();
550 ShenandoahAdjustPointersClosure cl;
551 ShenandoahAdjustPointersObjectClosure obj_cl(&cl);
552 while (r != NULL) {
553 if (! r->is_humongous_continuation()) {
554 heap->marked_object_iterate(r, &obj_cl);
555 }
556 r = _regions->claim_next();
557 }
558 }
559 };
560
561 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
562 private:
563 ShenandoahRootProcessor* _rp;
564
565 public:
566
567 ShenandoahAdjustRootPointersTask(ShenandoahRootProcessor* rp) :
568 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
569 _rp(rp) {
570 }
571
572 void work(uint worker_id) {
573 ShenandoahAdjustPointersClosure cl;
574 CLDToOopClosure adjust_cld_closure(&cl, true);
575 MarkingCodeBlobClosure adjust_code_closure(&cl,
576 CodeBlobToOopClosure::FixRelocations);
577
578 _rp->process_all_roots(&cl, &cl,
579 &adjust_cld_closure,
580 &adjust_code_closure, worker_id);
581 }
582 };
583
584 void ShenandoahMarkCompact::phase3_update_references() {
585 GCTraceTime(Info, gc, phases) time("Phase 2: Adjust pointers", _gc_timer);
586 ShenandoahHeap* heap = ShenandoahHeap::heap();
587
588 // Need cleared claim bits for the roots processing
589 ClassLoaderDataGraph::clear_claimed_marks();
590
591 WorkGang* workers = heap->workers();
592 uint nworkers = workers->active_workers();
593 {
594 COMPILER2_PRESENT(DerivedPointerTable::clear());
595
596 ShenandoahRootProcessor rp(heap, nworkers);
597 ShenandoahAdjustRootPointersTask task(&rp);
598 workers->run_task(&task);
599 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
600 }
601
602 ShenandoahHeapRegionSet* regions = heap->regions();
603 regions->clear_current_index();
604 ShenandoahAdjustPointersTask adjust_pointers_task(regions);
605 workers->run_task(&adjust_pointers_task);
606 }
|
210 }
211
212 _heap->set_bytes_allocated_since_cm(0);
213
214 _heap->set_need_update_refs(false);
215
216 _heap->set_full_gc_in_progress(false);
217 }
218
219 _gc_timer->register_gc_end();
220
221 policy->record_full_gc();
222
223 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_heapdumps);
224 _heap->post_full_gc_dump(_gc_timer);
225 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_heapdumps);
226
227 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc);
228
229 oopDesc::set_bs(old_bs);
230
231 if (UseShenandoahMatrix) {
232 if (PrintShenandoahMatrix) {
233 outputStream* log = Log(gc)::info_stream();
234 _heap->connection_matrix()->print_on(log);
235 }
236 }
237 }
238
239 #ifdef ASSERT
240 class VerifyNotForwardedPointersClosure : public MetadataAwareOopClosure {
241 private:
242 template <class T>
243 inline void do_oop_work(T* p) {
244 T o = oopDesc::load_heap_oop(p);
245 if (! oopDesc::is_null(o)) {
246 oop obj = oopDesc::decode_heap_oop_not_null(o);
247 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
248 "expect forwarded oop");
249 ShenandoahHeap* heap = ShenandoahHeap::heap();
250 if (! heap->is_marked_complete(obj)) {
251 tty->print_cr("ref region humongous? %s", BOOL_TO_STR(heap->heap_region_containing(p)->is_humongous()));
252 }
253 assert(heap->is_marked_complete(obj), "must be marked");
254 assert(! heap->allocated_after_complete_mark_start((HeapWord*) obj), "must be truly marked");
255 }
256 }
295 ShenandoahConcurrentMark* cm = _heap->concurrentMark();
296
297 cm->set_process_references(true);
298 cm->set_unload_classes(true);
299
300 ReferenceProcessor* rp = _heap->ref_processor();
301 // enable ("weak") refs discovery
302 rp->enable_discovery(true /*verify_no_refs*/);
303 rp->setup_policy(true); // snapshot the soft ref policy to be used in this cycle
304 rp->set_active_mt_degree(_heap->workers()->active_workers());
305
306 COMPILER2_PRESENT(DerivedPointerTable::clear());
307 cm->update_roots();
308 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
309
310 cm->mark_roots();
311 cm->shared_finish_mark_from_roots(/* full_gc = */ true);
312
313 _heap->swap_mark_bitmaps();
314
315 if (VerifyDuringGC) {
316 HandleMark hm; // handle scope
317 // Universe::heap()->prepare_for_verify();
318 _heap->prepare_for_verify();
319 // Note: we can verify only the heap here. When an object is
320 // marked, the previous value of the mark word (including
321 // identity hash values, ages, etc) is preserved, and the mark
322 // word is set to markOop::marked_value - effectively removing
323 // any hash values from the mark word. These hash values are
324 // used when verifying the dictionaries and so removing them
325 // from the mark word can make verification of the dictionaries
326 // fail. At the end of the GC, the original mark word values
327 // (including hash values) are restored to the appropriate
328 // objects.
329 // Universe::heap()->verify(VerifySilently, VerifyOption_G1UseMarkWord);
330 _heap->verify(VerifyOption_G1UseMarkWord);
331 }
332
333 #ifdef ASSERT
334 ShenandoahMCVerifyAfterMarkingRegionClosure cl;
474 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
475 ShenandoahHeap* heap = ShenandoahHeap::heap();
476
477 ShenandoahMCReclaimHumongousRegionClosure cl;
478 heap->heap_region_iterate(&cl);
479
480 // Initialize copy queues.
481 for (uint i = 0; i < heap->max_workers(); i++) {
482 copy_queues[i] = new ShenandoahHeapRegionSet(heap->max_regions());
483 }
484
485 ShenandoahHeapRegionSet* from_regions = heap->regions();
486 from_regions->clear_current_index();
487 ShenandoahPrepareForCompactionTask prepare_task(from_regions, copy_queues);
488 heap->workers()->run_task(&prepare_task);
489 }
490
491 class ShenandoahAdjustPointersClosure : public MetadataAwareOopClosure {
492 private:
493 ShenandoahHeap* _heap;
494 uint _from_idx;
495 public:
496
497 ShenandoahAdjustPointersClosure() : _heap(ShenandoahHeap::heap()) {
498 }
499
500 private:
501 template <class T>
502 inline void do_oop_work(T* p) {
503 T o = oopDesc::load_heap_oop(p);
504 if (! oopDesc::is_null(o)) {
505 oop obj = oopDesc::decode_heap_oop_not_null(o);
506 assert(_heap->is_marked_complete(obj), "must be marked");
507 oop forw = oop(BrooksPointer::get_raw(obj));
508 oopDesc::encode_store_heap_oop(p, forw);
509 if (UseShenandoahMatrix) {
510 if (_heap->is_in_reserved(p)) {
511 assert(_heap->is_in_reserved(forw), "must be in heap");
512 uint to_idx = _heap->heap_region_index_containing(forw);
513 _heap->connection_matrix()->set_connected(_from_idx, to_idx, true);
514 }
515 }
516 }
517 }
518 public:
519 void do_oop(oop* p) {
520 do_oop_work(p);
521 }
522 void do_oop(narrowOop* p) {
523 do_oop_work(p);
524 }
525 void set_from_idx(uint from_idx) {
526 _from_idx = from_idx;
527 }
528 };
529
530 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
531 private:
532 ShenandoahAdjustPointersClosure _cl;
533 ShenandoahHeap* _heap;
534 public:
535 ShenandoahAdjustPointersObjectClosure() :
536 _heap(ShenandoahHeap::heap()) {
537 }
538 void do_object(oop p) {
539 assert(_heap->is_marked_complete(p), "must be marked");
540 oop forw = oop(BrooksPointer::get_raw(p));
541 uint from_idx = _heap->heap_region_index_containing(forw);
542 _cl.set_from_idx(from_idx);
543 p->oop_iterate(&_cl);
544 }
545 };
546
547 class ShenandoahAdjustPointersTask : public AbstractGangTask {
548 private:
549 ShenandoahHeapRegionSet* _regions;
550 public:
551
552 ShenandoahAdjustPointersTask(ShenandoahHeapRegionSet* regions) :
553 AbstractGangTask("Shenandoah Adjust Pointers Task"),
554 _regions(regions) {
555 }
556
557 void work(uint worker_id) {
558 ShenandoahHeap* heap = ShenandoahHeap::heap();
559 ShenandoahHeapRegion* r = _regions->claim_next();
560 ShenandoahAdjustPointersObjectClosure obj_cl;
561 while (r != NULL) {
562 if (! r->is_humongous_continuation()) {
563 heap->marked_object_iterate(r, &obj_cl);
564 }
565 r = _regions->claim_next();
566 }
567 }
568 };
569
570 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
571 private:
572 ShenandoahRootProcessor* _rp;
573
574 public:
575
576 ShenandoahAdjustRootPointersTask(ShenandoahRootProcessor* rp) :
577 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
578 _rp(rp) {
579 }
580
581 void work(uint worker_id) {
582 ShenandoahAdjustPointersClosure cl;
583 CLDToOopClosure adjust_cld_closure(&cl, true);
584 MarkingCodeBlobClosure adjust_code_closure(&cl,
585 CodeBlobToOopClosure::FixRelocations);
586
587 _rp->process_all_roots(&cl, &cl,
588 &adjust_cld_closure,
589 &adjust_code_closure, worker_id);
590 }
591 };
592
593 void ShenandoahMarkCompact::phase3_update_references() {
594 GCTraceTime(Info, gc, phases) time("Phase 2: Adjust pointers", _gc_timer);
595 ShenandoahHeap* heap = ShenandoahHeap::heap();
596
597 if (UseShenandoahMatrix) {
598 heap->connection_matrix()->clear_all();
599 }
600
601 // Need cleared claim bits for the roots processing
602 ClassLoaderDataGraph::clear_claimed_marks();
603
604 WorkGang* workers = heap->workers();
605 uint nworkers = workers->active_workers();
606 {
607 COMPILER2_PRESENT(DerivedPointerTable::clear());
608
609 ShenandoahRootProcessor rp(heap, nworkers);
610 ShenandoahAdjustRootPointersTask task(&rp);
611 workers->run_task(&task);
612 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
613 }
614
615 ShenandoahHeapRegionSet* regions = heap->regions();
616 regions->clear_current_index();
617 ShenandoahAdjustPointersTask adjust_pointers_task(regions);
618 workers->run_task(&adjust_pointers_task);
619 }
|