6639
6640 VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_free_list);
6641 heap_region_iterate(&cl);
6642
6643 _old_set.verify_end();
6644 _humongous_set.verify_end();
6645 _free_list.verify_end();
6646 }
6647
6648 // Optimized nmethod scanning
6649
6650 class RegisterNMethodOopClosure: public OopClosure {
6651 G1CollectedHeap* _g1h;
6652 nmethod* _nm;
6653
6654 template <class T> void do_oop_work(T* p) {
6655 T heap_oop = oopDesc::load_heap_oop(p);
6656 if (!oopDesc::is_null(heap_oop)) {
6657 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
6658 HeapRegion* hr = _g1h->heap_region_containing(obj);
6659 assert(!hr->isHumongous(), "code root in humongous region?");
6660
6661 // HeapRegion::add_strong_code_root() avoids adding duplicate
6662 // entries but having duplicates is OK since we "mark" nmethods
6663 // as visited when we scan the strong code root lists during the GC.
6664 hr->add_strong_code_root(_nm);
6665 assert(hr->rem_set()->strong_code_roots_list_contains(_nm), "add failed?");
6666 }
6667 }
6668
6669 public:
6670 RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
6671 _g1h(g1h), _nm(nm) {}
6672
6673 void do_oop(oop* p) { do_oop_work(p); }
6674 void do_oop(narrowOop* p) { do_oop_work(p); }
6675 };
6676
6677 class UnregisterNMethodOopClosure: public OopClosure {
6678 G1CollectedHeap* _g1h;
6679 nmethod* _nm;
6680
6681 template <class T> void do_oop_work(T* p) {
6682 T heap_oop = oopDesc::load_heap_oop(p);
6683 if (!oopDesc::is_null(heap_oop)) {
6684 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
6685 HeapRegion* hr = _g1h->heap_region_containing(obj);
6686 assert(!hr->isHumongous(), "code root in humongous region?");
6687 hr->remove_strong_code_root(_nm);
6688 assert(!hr->rem_set()->strong_code_roots_list_contains(_nm), "remove failed?");
6689 }
6690 }
6691
6692 public:
6693 UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
6694 _g1h(g1h), _nm(nm) {}
6695
6696 void do_oop(oop* p) { do_oop_work(p); }
6697 void do_oop(narrowOop* p) { do_oop_work(p); }
6698 };
6699
6700 void G1CollectedHeap::register_nmethod(nmethod* nm) {
6701 CollectedHeap::register_nmethod(nm);
6702
6703 guarantee(nm != NULL, "sanity");
6704 RegisterNMethodOopClosure reg_cl(this, nm);
6705 nm->oops_do(®_cl);
6706 }
6707
6708 void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
6709 CollectedHeap::unregister_nmethod(nm);
6710
6711 guarantee(nm != NULL, "sanity");
6712 UnregisterNMethodOopClosure reg_cl(this, nm);
6713 nm->oops_do(®_cl, true);
6714 }
6715
6716 class MigrateCodeRootsHeapRegionClosure: public HeapRegionClosure {
6717 public:
6718 bool doHeapRegion(HeapRegion *hr) {
6719 assert(!hr->isHumongous(), "humongous region in collection set?");
6720 hr->migrate_strong_code_roots();
6721 return false;
6722 }
6723 };
6724
6725 void G1CollectedHeap::migrate_strong_code_roots() {
6726 MigrateCodeRootsHeapRegionClosure cl;
6727 double migrate_start = os::elapsedTime();
6728 collection_set_iterate(&cl);
6729 double migration_time_ms = (os::elapsedTime() - migrate_start) * 1000.0;
6730 g1_policy()->phase_times()->record_strong_code_root_migration_time(migration_time_ms);
6731 }
6732
6733 // Mark all the code roots that point into regions *not* in the
6734 // collection set.
6735 //
6736 // Note we do not want to use a "marking" CodeBlobToOopClosure while
6737 // walking the the code roots lists of regions not in the collection
6738 // set. Suppose we have an nmethod (M) that points to objects in two
6739 // separate regions - one in the collection set (R1) and one not (R2).
6779 _oop_cl(cm, hr, worker_id) {}
6780
6781 void do_code_blob(CodeBlob* cb) {
6782 nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null();
6783 if (nm != NULL) {
6784 nm->oops_do(&_oop_cl);
6785 }
6786 }
6787 };
6788
6789 class MarkStrongCodeRootsHRClosure: public HeapRegionClosure {
6790 G1CollectedHeap* _g1h;
6791 uint _worker_id;
6792
6793 public:
6794 MarkStrongCodeRootsHRClosure(G1CollectedHeap* g1h, uint worker_id) :
6795 _g1h(g1h), _worker_id(worker_id) {}
6796
6797 bool doHeapRegion(HeapRegion *hr) {
6798 HeapRegionRemSet* hrrs = hr->rem_set();
6799 if (hr->isHumongous()) {
6800 // Code roots should never be attached to a humongous region
6801 assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
6802 return false;
6803 }
6804
6805 if (hr->in_collection_set()) {
6806 // Don't mark code roots into regions in the collection set here.
6807 // They will be marked when we scan them.
6808 return false;
6809 }
6810
6811 MarkStrongCodeRootCodeBlobClosure cb_cl(_g1h->concurrent_mark(), hr, _worker_id);
6812 hr->strong_code_roots_do(&cb_cl);
6813 return false;
6814 }
6815 };
6816
6817 void G1CollectedHeap::mark_strong_code_roots(uint worker_id) {
6818 MarkStrongCodeRootsHRClosure cl(this, worker_id);
6819 if (G1CollectedHeap::use_parallel_gc_threads()) {
6820 heap_region_par_iterate_chunked(&cl,
6821 worker_id,
|
6639
6640 VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_free_list);
6641 heap_region_iterate(&cl);
6642
6643 _old_set.verify_end();
6644 _humongous_set.verify_end();
6645 _free_list.verify_end();
6646 }
6647
6648 // Optimized nmethod scanning
6649
6650 class RegisterNMethodOopClosure: public OopClosure {
6651 G1CollectedHeap* _g1h;
6652 nmethod* _nm;
6653
6654 template <class T> void do_oop_work(T* p) {
6655 T heap_oop = oopDesc::load_heap_oop(p);
6656 if (!oopDesc::is_null(heap_oop)) {
6657 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
6658 HeapRegion* hr = _g1h->heap_region_containing(obj);
6659 assert(!hr->continuesHumongous(),
6660 err_msg("trying to add code root "PTR_FORMAT" in continuation of humongous region "HR_FORMAT
6661 " starting at "HR_FORMAT,
6662 _nm, HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region())));
6663
6664 // HeapRegion::add_strong_code_root() avoids adding duplicate
6665 // entries but having duplicates is OK since we "mark" nmethods
6666 // as visited when we scan the strong code root lists during the GC.
6667 hr->add_strong_code_root(_nm);
6668 assert(hr->rem_set()->strong_code_roots_list_contains(_nm),
6669 err_msg("failed to add code root "PTR_FORMAT" to remembered set of region "HR_FORMAT,
6670 _nm, HR_FORMAT_PARAMS(hr)));
6671 }
6672 }
6673
6674 public:
6675 RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
6676 _g1h(g1h), _nm(nm) {}
6677
6678 void do_oop(oop* p) { do_oop_work(p); }
6679 void do_oop(narrowOop* p) { do_oop_work(p); }
6680 };
6681
6682 class UnregisterNMethodOopClosure: public OopClosure {
6683 G1CollectedHeap* _g1h;
6684 nmethod* _nm;
6685
6686 template <class T> void do_oop_work(T* p) {
6687 T heap_oop = oopDesc::load_heap_oop(p);
6688 if (!oopDesc::is_null(heap_oop)) {
6689 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
6690 HeapRegion* hr = _g1h->heap_region_containing(obj);
6691 assert(!hr->continuesHumongous(),
6692 err_msg("trying to remove code root "PTR_FORMAT" in continuation of humongous region "HR_FORMAT
6693 " starting at "HR_FORMAT,
6694 _nm, HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region())));
6695
6696 hr->remove_strong_code_root(_nm);
6697 assert(!hr->rem_set()->strong_code_roots_list_contains(_nm),
6698 err_msg("failed to remove code root "PTR_FORMAT" of region "HR_FORMAT,
6699 _nm, HR_FORMAT_PARAMS(hr)));
6700 }
6701 }
6702
6703 public:
6704 UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
6705 _g1h(g1h), _nm(nm) {}
6706
6707 void do_oop(oop* p) { do_oop_work(p); }
6708 void do_oop(narrowOop* p) { do_oop_work(p); }
6709 };
6710
6711 void G1CollectedHeap::register_nmethod(nmethod* nm) {
6712 CollectedHeap::register_nmethod(nm);
6713
6714 guarantee(nm != NULL, "sanity");
6715 RegisterNMethodOopClosure reg_cl(this, nm);
6716 nm->oops_do(®_cl);
6717 }
6718
6719 void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
6720 CollectedHeap::unregister_nmethod(nm);
6721
6722 guarantee(nm != NULL, "sanity");
6723 UnregisterNMethodOopClosure reg_cl(this, nm);
6724 nm->oops_do(®_cl, true);
6725 }
6726
6727 class MigrateCodeRootsHeapRegionClosure: public HeapRegionClosure {
6728 public:
6729 bool doHeapRegion(HeapRegion *hr) {
6730 assert(!hr->isHumongous(),
6731 err_msg("humongous region "HR_FORMAT" should not have been added to collection set",
6732 HR_FORMAT_PARAMS(hr)));
6733 hr->migrate_strong_code_roots();
6734 return false;
6735 }
6736 };
6737
6738 void G1CollectedHeap::migrate_strong_code_roots() {
6739 MigrateCodeRootsHeapRegionClosure cl;
6740 double migrate_start = os::elapsedTime();
6741 collection_set_iterate(&cl);
6742 double migration_time_ms = (os::elapsedTime() - migrate_start) * 1000.0;
6743 g1_policy()->phase_times()->record_strong_code_root_migration_time(migration_time_ms);
6744 }
6745
6746 // Mark all the code roots that point into regions *not* in the
6747 // collection set.
6748 //
6749 // Note we do not want to use a "marking" CodeBlobToOopClosure while
6750 // walking the the code roots lists of regions not in the collection
6751 // set. Suppose we have an nmethod (M) that points to objects in two
6752 // separate regions - one in the collection set (R1) and one not (R2).
6792 _oop_cl(cm, hr, worker_id) {}
6793
6794 void do_code_blob(CodeBlob* cb) {
6795 nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null();
6796 if (nm != NULL) {
6797 nm->oops_do(&_oop_cl);
6798 }
6799 }
6800 };
6801
6802 class MarkStrongCodeRootsHRClosure: public HeapRegionClosure {
6803 G1CollectedHeap* _g1h;
6804 uint _worker_id;
6805
6806 public:
6807 MarkStrongCodeRootsHRClosure(G1CollectedHeap* g1h, uint worker_id) :
6808 _g1h(g1h), _worker_id(worker_id) {}
6809
6810 bool doHeapRegion(HeapRegion *hr) {
6811 HeapRegionRemSet* hrrs = hr->rem_set();
6812 if (hr->continuesHumongous()) {
6813 // Code roots should never be attached to a continuation of a humongous region
6814 assert(hrrs->strong_code_roots_list_length() == 0,
6815 err_msg("code roots should never be attached to continuations of humongous region "HR_FORMAT
6816 " starting at "HR_FORMAT", but has "INT32_FORMAT,
6817 HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()),
6818 hrrs->strong_code_roots_list_length()));
6819 return false;
6820 }
6821
6822 if (hr->in_collection_set()) {
6823 // Don't mark code roots into regions in the collection set here.
6824 // They will be marked when we scan them.
6825 return false;
6826 }
6827
6828 MarkStrongCodeRootCodeBlobClosure cb_cl(_g1h->concurrent_mark(), hr, _worker_id);
6829 hr->strong_code_roots_do(&cb_cl);
6830 return false;
6831 }
6832 };
6833
6834 void G1CollectedHeap::mark_strong_code_roots(uint worker_id) {
6835 MarkStrongCodeRootsHRClosure cl(this, worker_id);
6836 if (G1CollectedHeap::use_parallel_gc_threads()) {
6837 heap_region_par_iterate_chunked(&cl,
6838 worker_id,
|