--- old/src/share/vm/gc_implementation/g1/concurrentMark.cpp 2013-06-26 14:23:56.578750626 -0700 +++ new/src/share/vm/gc_implementation/g1/concurrentMark.cpp 2013-06-26 14:23:56.358560686 -0700 @@ -4516,7 +4516,7 @@ _total_prev_live_bytes(0), _total_next_live_bytes(0), _hum_used_bytes(0), _hum_capacity_bytes(0), _hum_prev_live_bytes(0), _hum_next_live_bytes(0), - _total_remset_bytes(0) { + _total_remset_bytes(0), _total_strong_code_roots_bytes(0) { G1CollectedHeap* g1h = G1CollectedHeap::heap(); MemRegion g1_committed = g1h->g1_committed(); MemRegion g1_reserved = g1h->g1_reserved(); @@ -4540,9 +4540,11 @@ G1PPRL_BYTE_H_FORMAT G1PPRL_BYTE_H_FORMAT G1PPRL_DOUBLE_H_FORMAT + G1PPRL_BYTE_H_FORMAT G1PPRL_BYTE_H_FORMAT, "type", "address-range", - "used", "prev-live", "next-live", "gc-eff", "remset"); + "used", "prev-live", "next-live", "gc-eff", + "remset", "code-roots"); _out->print_cr(G1PPRL_LINE_PREFIX G1PPRL_TYPE_H_FORMAT G1PPRL_ADDR_BASE_H_FORMAT @@ -4550,9 +4552,11 @@ G1PPRL_BYTE_H_FORMAT G1PPRL_BYTE_H_FORMAT G1PPRL_DOUBLE_H_FORMAT + G1PPRL_BYTE_H_FORMAT G1PPRL_BYTE_H_FORMAT, "", "", - "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)", "(bytes)"); + "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)", + "(bytes)", "(bytes)"); } // It takes as a parameter a reference to one of the _hum_* fields, it @@ -4595,6 +4599,8 @@ size_t next_live_bytes = r->next_live_bytes(); double gc_eff = r->gc_efficiency(); size_t remset_bytes = r->rem_set()->mem_size(); + size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size(); + if (r->used() == 0) { type = "FREE"; } else if (r->is_survivor()) { @@ -4629,6 +4635,7 @@ _total_prev_live_bytes += prev_live_bytes; _total_next_live_bytes += next_live_bytes; _total_remset_bytes += remset_bytes; + _total_strong_code_roots_bytes += strong_code_roots_bytes; // Print a line for this particular region. _out->print_cr(G1PPRL_LINE_PREFIX @@ -4638,9 +4645,11 @@ G1PPRL_BYTE_FORMAT G1PPRL_BYTE_FORMAT G1PPRL_DOUBLE_FORMAT + G1PPRL_BYTE_FORMAT G1PPRL_BYTE_FORMAT, type, bottom, end, - used_bytes, prev_live_bytes, next_live_bytes, gc_eff , remset_bytes); + used_bytes, prev_live_bytes, next_live_bytes, gc_eff, + remset_bytes, strong_code_roots_bytes); return false; } @@ -4656,7 +4665,8 @@ G1PPRL_SUM_MB_PERC_FORMAT("used") G1PPRL_SUM_MB_PERC_FORMAT("prev-live") G1PPRL_SUM_MB_PERC_FORMAT("next-live") - G1PPRL_SUM_MB_FORMAT("remset"), + G1PPRL_SUM_MB_FORMAT("remset") + G1PPRL_SUM_MB_FORMAT("code-roots"), bytes_to_mb(_total_capacity_bytes), bytes_to_mb(_total_used_bytes), perc(_total_used_bytes, _total_capacity_bytes), @@ -4664,6 +4674,7 @@ perc(_total_prev_live_bytes, _total_capacity_bytes), bytes_to_mb(_total_next_live_bytes), perc(_total_next_live_bytes, _total_capacity_bytes), - bytes_to_mb(_total_remset_bytes)); + bytes_to_mb(_total_remset_bytes), + bytes_to_mb(_total_strong_code_roots_bytes)); _out->cr(); } --- old/src/share/vm/gc_implementation/g1/concurrentMark.hpp 2013-06-26 14:23:57.790536480 -0700 +++ new/src/share/vm/gc_implementation/g1/concurrentMark.hpp 2013-06-26 14:23:57.574789448 -0700 @@ -1257,6 +1257,9 @@ // Accumulator for the remembered set size size_t _total_remset_bytes; + // Accumulator for strong code roots memory size + size_t _total_strong_code_roots_bytes; + static double perc(size_t val, size_t total) { if (total == 0) { return 0.0; --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2013-06-26 14:23:58.964410070 -0700 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2013-06-26 14:23:58.737749005 -0700 @@ -1176,14 +1176,16 @@ _g1h(g1h), _mr_bs(mr_bs) {} bool doHeapRegion(HeapRegion* r) { + HeapRegionRemSet* hrrs = r->rem_set(); + if (r->continuesHumongous()) { // We'll assert that the strong code root list is empty - assert(r->strong_code_root_list()->is_empty(), "sanity"); + assert(hrrs->strong_code_roots_list_length() == 0, "sanity"); return false; } + _g1h->reset_gc_time_stamps(r); - HeapRegionRemSet* hrrs = r->rem_set(); - if (hrrs != NULL) hrrs->clear(); + hrrs->clear(); // You might think here that we could clear just the cards // corresponding to the used region. But no: if we leave a dirty card // in a region we might allocate into, then it would prevent that card @@ -1191,9 +1193,6 @@ // Re: the performance cost: we shouldn't be doing full GC anyway! _mr_bs->clear(MemRegion(r->bottom(), r->end())); - // We'll also clear the strong code root list for this region - r->strong_code_root_list()->clear(); - return false; } }; @@ -3107,9 +3106,10 @@ // Now fetch the region containing the object HeapRegion* hr = _g1h->heap_region_containing(obj); + HeapRegionRemSet* hrrs = hr->rem_set(); // Verify that the strong code root list for this region // contains the nmethod - if (!hr->strong_code_root_list()->contains(_nm)) { + if (!hrrs->strong_code_roots_list_contains(_nm)) { gclog_or_tty->print_cr("Code root location "PTR_FORMAT" " "from nmethod "PTR_FORMAT" not in strong " "code roots for region ["PTR_FORMAT","PTR_FORMAT")", @@ -6558,7 +6558,7 @@ // entries but having duplicates is OK since we "mark" nmethods // as visited when we scan the strong code root lists during the GC. hr->add_strong_code_root(_nm); - assert(hr->strong_code_root_list()->contains(_nm), "push failed?"); + assert(hr->rem_set()->strong_code_roots_list_contains(_nm), "add failed?"); } } @@ -6581,7 +6581,7 @@ HeapRegion* hr = _g1h->heap_region_containing(obj); assert(!hr->isHumongous(), "code root in humongous region?"); hr->remove_strong_code_root(_nm); - assert(!hr->strong_code_root_list()->contains(_nm), "remove failed?"); + assert(!hr->rem_set()->strong_code_roots_list_contains(_nm), "remove failed?"); } } @@ -6687,9 +6687,10 @@ _g1h(g1h), _worker_id(worker_id) {} bool doHeapRegion(HeapRegion *hr) { + HeapRegionRemSet* hrrs = hr->rem_set(); if (hr->isHumongous()) { // Code roots should never be attached to a humongous region - assert(hr->strong_code_root_list()->is_empty(), "sanity"); + assert(hrrs->strong_code_roots_list_length() == 0, "sanity"); return false; } --- old/src/share/vm/gc_implementation/g1/g1RemSetSummary.cpp 2013-06-26 14:24:00.191679600 -0700 +++ new/src/share/vm/gc_implementation/g1/g1RemSetSummary.cpp 2013-06-26 14:23:59.974693535 -0700 @@ -127,32 +127,55 @@ class HRRSStatsIter: public HeapRegionClosure { size_t _occupied; - size_t _total_mem_sz; - size_t _max_mem_sz; - HeapRegion* _max_mem_sz_region; + + size_t _total_rs_mem_sz; + size_t _max_rs_mem_sz; + HeapRegion* _max_rs_mem_sz_region; + + size_t _total_code_root_mem_sz; + size_t _max_code_root_mem_sz; + HeapRegion* _max_code_root_mem_sz_region; public: HRRSStatsIter() : _occupied(0), - _total_mem_sz(0), - _max_mem_sz(0), - _max_mem_sz_region(NULL) + _total_rs_mem_sz(0), + _max_rs_mem_sz(0), + _max_rs_mem_sz_region(NULL), + _total_code_root_mem_sz(0), + _max_code_root_mem_sz(0), + _max_code_root_mem_sz_region(NULL) {} bool doHeapRegion(HeapRegion* r) { - size_t mem_sz = r->rem_set()->mem_size(); - if (mem_sz > _max_mem_sz) { - _max_mem_sz = mem_sz; - _max_mem_sz_region = r; + HeapRegionRemSet* hrrs = r->rem_set(); + + // HeapRegionRemSet::mem_size() includes the + // size of the strong code roots + size_t rs_mem_sz = hrrs->mem_size(); + if (rs_mem_sz > _max_rs_mem_sz) { + _max_rs_mem_sz = rs_mem_sz; + _max_rs_mem_sz_region = r; + } + _total_rs_mem_sz += rs_mem_sz; + + size_t code_root_mem_sz = hrrs->strong_code_roots_mem_size(); + if (code_root_mem_sz > _max_code_root_mem_sz) { + _max_code_root_mem_sz = code_root_mem_sz; + _max_code_root_mem_sz_region = r; } - _total_mem_sz += mem_sz; - size_t occ = r->rem_set()->occupied(); + _total_code_root_mem_sz += code_root_mem_sz; + + size_t occ = hrrs->occupied(); _occupied += occ; return false; } - size_t total_mem_sz() { return _total_mem_sz; } - size_t max_mem_sz() { return _max_mem_sz; } + size_t total_rs_mem_sz() { return _total_rs_mem_sz; } + size_t max_rs_mem_sz() { return _max_rs_mem_sz; } + HeapRegion* max_rs_mem_sz_region() { return _max_rs_mem_sz_region; } + size_t total_code_root_mem_sz() { return _total_code_root_mem_sz; } + size_t max_code_root_mem_sz() { return _max_code_root_mem_sz; } + HeapRegion* max_code_root_mem_sz_region() { return _max_code_root_mem_sz_region; } size_t occupied() { return _occupied; } - HeapRegion* max_mem_sz_region() { return _max_mem_sz_region; } }; double calc_percentage(size_t numerator, size_t denominator) { @@ -184,22 +207,33 @@ HRRSStatsIter blk; G1CollectedHeap::heap()->heap_region_iterate(&blk); + // RemSet stats out->print_cr(" Total heap region rem set sizes = "SIZE_FORMAT"K." " Max = "SIZE_FORMAT"K.", - blk.total_mem_sz()/K, blk.max_mem_sz()/K); + blk.total_rs_mem_sz()/K, blk.max_rs_mem_sz()/K); out->print_cr(" Static structures = "SIZE_FORMAT"K," " free_lists = "SIZE_FORMAT"K.", HeapRegionRemSet::static_mem_size() / K, HeapRegionRemSet::fl_mem_size() / K); out->print_cr(" "SIZE_FORMAT" occupied cards represented.", blk.occupied()); - HeapRegion* max_mem_sz_region = blk.max_mem_sz_region(); - HeapRegionRemSet* rem_set = max_mem_sz_region->rem_set(); + HeapRegion* max_rs_mem_sz_region = blk.max_rs_mem_sz_region(); + HeapRegionRemSet* max_rs_rem_set = max_rs_mem_sz_region->rem_set(); out->print_cr(" Max size region = "HR_FORMAT", " "size = "SIZE_FORMAT "K, occupied = "SIZE_FORMAT"K.", - HR_FORMAT_PARAMS(max_mem_sz_region), - (rem_set->mem_size() + K - 1)/K, - (rem_set->occupied() + K - 1)/K); - + HR_FORMAT_PARAMS(max_rs_mem_sz_region), + (max_rs_rem_set->mem_size() + K - 1)/K, + (max_rs_rem_set->occupied() + K - 1)/K); out->print_cr(" Did %d coarsenings.", num_coarsenings()); + // Strong code root stats + out->print_cr(" Total heap region code-root set sizes = "SIZE_FORMAT"K." + " Max = "SIZE_FORMAT"K.", + blk.total_code_root_mem_sz()/K, blk.max_code_root_mem_sz()/K); + HeapRegion* max_code_root_mem_sz_region = blk.max_code_root_mem_sz_region(); + HeapRegionRemSet* max_code_root_rem_set = max_code_root_mem_sz_region->rem_set(); + out->print_cr(" Max size region = "HR_FORMAT", " + "size = "SIZE_FORMAT "K, num_elems = "SIZE_FORMAT".", + HR_FORMAT_PARAMS(max_code_root_mem_sz_region), + (max_code_root_rem_set->strong_code_roots_mem_size() + K - 1)/K, + (max_code_root_rem_set->strong_code_roots_list_length())); } --- old/src/share/vm/gc_implementation/g1/heapRegion.cpp 2013-06-26 14:24:01.302914463 -0700 +++ new/src/share/vm/gc_implementation/g1/heapRegion.cpp 2013-06-26 14:24:01.085033719 -0700 @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "code/nmethod.hpp" #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/g1OopClosures.inline.hpp" @@ -32,7 +33,6 @@ #include "memory/genOopClosures.inline.hpp" #include "memory/iterator.hpp" #include "oops/oop.inline.hpp" -#include "utilities/growableArray.hpp" int HeapRegion::LogOfHRGrainBytes = 0; int HeapRegion::LogOfHRGrainWords = 0; @@ -226,20 +226,13 @@ if (!par) { // If this is parallel, this will be done later. HeapRegionRemSet* hrrs = rem_set(); - if (hrrs != NULL) hrrs->clear(); + hrrs->clear(); _claimed = InitialClaimValue; } zero_marked_bytes(); _offsets.resize(HeapRegion::GrainWords); init_top_at_mark_start(); - - if (_strong_code_root_list != NULL) { - delete _strong_code_root_list; - } - _strong_code_root_list = new (ResourceObj::C_HEAP, mtGC) - GrowableArray(10, 0, NULL, true); - if (clear_space) clear(SpaceDecorator::Mangle); } @@ -368,8 +361,9 @@ #endif // ASSERT _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0), - _predicted_bytes_to_copy(0), _strong_code_root_list(NULL) + _predicted_bytes_to_copy(0) { + _rem_set = new HeapRegionRemSet(sharedOffsetArray, this); _orig_end = mr.end(); // Note that initialize() will set the start of the unmarked area of the // region. @@ -377,8 +371,6 @@ set_top(bottom()); set_saved_mark(); - _rem_set = new HeapRegionRemSet(sharedOffsetArray, this); - assert(HeapRegionRemSet::num_par_rem_sets() > 0, "Invariant."); } @@ -601,111 +593,26 @@ // Code roots support void HeapRegion::add_strong_code_root(nmethod* nm) { - assert(nm != NULL, "sanity"); - // Search for the code blob from the RHS to avoid - // duplicate entries as much as possible - if (_strong_code_root_list->find_from_end(nm) < 0) { - // Code blob isn't already in the list - _strong_code_root_list->push(nm); - } + HeapRegionRemSet* hrrs = rem_set(); + hrrs->add_strong_code_root(nm); } void HeapRegion::remove_strong_code_root(nmethod* nm) { - assert(nm != NULL, "sanity"); - int idx = _strong_code_root_list->find(nm); - while (idx >= 0) { - _strong_code_root_list->remove_at(idx); - idx = _strong_code_root_list->find(nm); - } + HeapRegionRemSet* hrrs = rem_set(); + hrrs->remove_strong_code_root(nm); } -class NMethodMigrationOopClosure : public OopClosure { - G1CollectedHeap* _g1h; - HeapRegion* _from; - nmethod* _nm; - - uint _num_self_forwarded; - - template void do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_from->is_in(obj)) { - // Reference still points into the source region. - // Since roots are immediately evacuated this means that - // we must have self forwarded the object - assert(obj->is_forwarded(), - err_msg("code roots should be immediately evacuated. " - "Ref: "PTR_FORMAT", " - "Obj: "PTR_FORMAT", " - "Region: "HR_FORMAT, - p, (void*) obj, HR_FORMAT_PARAMS(_from))); - assert(obj->forwardee() == obj, - err_msg("not self forwarded? obj = "PTR_FORMAT, (void*)obj)); - - // The object has been self forwarded. - // Note, if we're during an initial mark pause, there is - // no need to explicitly mark object. It will be marked - // during the regular evacuation failure handling code. - _num_self_forwarded++; - } else { - // The reference points into a promotion or to-space region - HeapRegion* to = _g1h->heap_region_containing(obj); - to->add_strong_code_root(_nm); - } - } - } - -public: - NMethodMigrationOopClosure(G1CollectedHeap* g1h, HeapRegion* from, nmethod* nm): - _g1h(g1h), _from(from), _nm(nm), _num_self_forwarded(0) {} - - void do_oop(narrowOop* p) { do_oop_work(p); } - void do_oop(oop* p) { do_oop_work(p); } - - uint retain() { return _num_self_forwarded > 0; } -}; - void HeapRegion::migrate_strong_code_roots() { assert(in_collection_set(), "only collection set regions"); assert(!isHumongous(), "not humongous regions"); - ResourceMark rm; - - // List of code blobs to retain for this region - GrowableArray to_be_retained(10); - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - while (strong_code_root_list()->is_nonempty()) { - nmethod *nm = strong_code_root_list()->pop(); - if (nm != NULL) { - NMethodMigrationOopClosure oop_cl(g1h, this, nm); - nm->oops_do(&oop_cl); - if (oop_cl.retain()) { - to_be_retained.push(nm); - } - } - } - - // Now push any code roots we need to retain - // FIXME: assert that region got an evacuation failure if non-empty - while (to_be_retained.is_nonempty()) { - nmethod* nm = to_be_retained.pop(); - assert(nm != NULL, "sanity"); - add_strong_code_root(nm); - } + HeapRegionRemSet* hrrs = rem_set(); + hrrs->migrate_strong_code_roots(); } void HeapRegion::strong_code_roots_do(CodeBlobClosure* blk) const { - for (int i = 0; i < _strong_code_root_list->length(); i += 1) { - nmethod* nm = _strong_code_root_list->at(i); - blk->do_code_blob(nm); - } -} - -size_t HeapRegion::strong_code_root_mem_size() { - return sizeof(GrowableArray) + - _strong_code_root_list->max_length() * sizeof(nmethod*); + HeapRegionRemSet* hrrs = rem_set(); + hrrs->strong_code_roots_do(blk); } class VerifyStrongCodeRootOopClosure: public OopClosure { @@ -803,13 +710,16 @@ return; } + HeapRegionRemSet* hrrs = rem_set(); + int strong_code_roots_length = hrrs->strong_code_roots_list_length(); + // if this region is empty then there should be no entries // on its strong code root list if (is_empty()) { - if (!_strong_code_root_list->is_empty()) { + if (strong_code_roots_length > 0) { gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is empty " "but has "INT32_FORMAT" code root entries", - bottom(), end(), _strong_code_root_list->length()); + bottom(), end(), strong_code_roots_length); *failures = true; } return; @@ -817,10 +727,10 @@ // An H-region should have an empty strong code root list if (isHumongous()) { - if (!_strong_code_root_list->is_empty()) { + if (strong_code_roots_length > 0) { gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is humongous " "but has "INT32_FORMAT" code root entries", - bottom(), end(), _strong_code_root_list->length()); + bottom(), end(), strong_code_roots_length); *failures = true; } return; --- old/src/share/vm/gc_implementation/g1/heapRegion.hpp 2013-06-26 14:24:03.042270129 -0700 +++ new/src/share/vm/gc_implementation/g1/heapRegion.hpp 2013-06-26 14:24:02.660779867 -0700 @@ -338,9 +338,6 @@ // the total value for the collection set. size_t _predicted_bytes_to_copy; - // A list of code blobs (nmethods) whose code contains pointers into this region - GrowableArray* _strong_code_root_list; - public: HeapRegion(uint hrs_index, G1BlockOffsetSharedArray* sharedOffsetArray, @@ -801,31 +798,23 @@ virtual void reset_after_compaction(); - // Routines for managing the list of code roots that point into - // this heap region. + // Routines for managing a list of code roots (attached to the + // this region's RSet) that point into this heap region. void add_strong_code_root(nmethod* nm); void remove_strong_code_root(nmethod* nm); - GrowableArray* strong_code_root_list() { - return _strong_code_root_list; - } - - // During a collection, migrate successfully evacuated strong - // code roots attached to this region to the new regions that - // they point into. Unsuccessfully evacuated code roots are - // not migrated. + // During a collection, migrate the successfully evacuated + // strong code roots that referenced into this region to the + // new regions that they now point into. Unsuccessfully + // evacuated code roots are not migrated. void migrate_strong_code_roots(); // Applies blk->do_code_blob() to each of the entries in - // the strong code roots list; + // the strong code roots list for this region void strong_code_roots_do(CodeBlobClosure* blk) const; - // Returns the amount of memory, in bytes, currently - // consumed by the strong code roots. - size_t strong_code_root_mem_size(); - - // Verify that the entries on the strong code root list are live and - // include at least one pointer into this region. + // Verify that the entries on the strong code root list for this + // region are live and include at least one pointer into this region. void verify_strong_code_roots(VerifyOption vo, bool* failures) const; void print() const; --- old/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp 2013-06-26 14:24:04.947135421 -0700 +++ new/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp 2013-06-26 14:24:04.598976540 -0700 @@ -33,6 +33,7 @@ #include "oops/oop.inline.hpp" #include "utilities/bitMap.inline.hpp" #include "utilities/globalDefinitions.hpp" +#include "utilities/growableArray.hpp" class PerRegionTable: public CHeapObj { friend class OtherRegionsTable; @@ -849,7 +850,7 @@ HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, HeapRegion* hr) - : _bosa(bosa), _other_regions(hr) { + : _bosa(bosa), _strong_code_roots_list(NULL), _other_regions(hr) { reset_for_par_iteration(); } @@ -908,6 +909,12 @@ } void HeapRegionRemSet::clear() { + if (_strong_code_roots_list != NULL) { + delete _strong_code_roots_list; + } + _strong_code_roots_list = new (ResourceObj::C_HEAP, mtGC) + GrowableArray(10, 0, NULL, true); + _other_regions.clear(); assert(occupied() == 0, "Should be clear."); reset_for_par_iteration(); @@ -925,6 +932,118 @@ _other_regions.scrub(ctbs, region_bm, card_bm); } + +// Code roots support + +void HeapRegionRemSet::add_strong_code_root(nmethod* nm) { + assert(nm != NULL, "sanity"); + // Search for the code blob from the RHS to avoid + // duplicate entries as much as possible + if (_strong_code_roots_list->find_from_end(nm) < 0) { + // Code blob isn't already in the list + _strong_code_roots_list->push(nm); + } +} + +void HeapRegionRemSet::remove_strong_code_root(nmethod* nm) { + assert(nm != NULL, "sanity"); + int idx = _strong_code_roots_list->find(nm); + if (idx >= 0) { + _strong_code_roots_list->remove_at(idx); + } + // Check that there were no duplicates + guarantee(_strong_code_roots_list->find(nm) < 0, "duplicate entry found"); +} + +class NMethodMigrationOopClosure : public OopClosure { + G1CollectedHeap* _g1h; + HeapRegion* _from; + nmethod* _nm; + + uint _num_self_forwarded; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (_from->is_in(obj)) { + // Reference still points into the source region. + // Since roots are immediately evacuated this means that + // we must have self forwarded the object + assert(obj->is_forwarded(), + err_msg("code roots should be immediately evacuated. " + "Ref: "PTR_FORMAT", " + "Obj: "PTR_FORMAT", " + "Region: "HR_FORMAT, + p, (void*) obj, HR_FORMAT_PARAMS(_from))); + assert(obj->forwardee() == obj, + err_msg("not self forwarded? obj = "PTR_FORMAT, (void*)obj)); + + // The object has been self forwarded. + // Note, if we're during an initial mark pause, there is + // no need to explicitly mark object. It will be marked + // during the regular evacuation failure handling code. + _num_self_forwarded++; + } else { + // The reference points into a promotion or to-space region + HeapRegion* to = _g1h->heap_region_containing(obj); + to->rem_set()->add_strong_code_root(_nm); + } + } + } + +public: + NMethodMigrationOopClosure(G1CollectedHeap* g1h, HeapRegion* from, nmethod* nm): + _g1h(g1h), _from(from), _nm(nm), _num_self_forwarded(0) {} + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } + + uint retain() { return _num_self_forwarded > 0; } +}; + +void HeapRegionRemSet::migrate_strong_code_roots() { + assert(hr()->in_collection_set(), "only collection set regions"); + assert(!hr()->isHumongous(), "not humongous regions"); + + ResourceMark rm; + + // List of code blobs to retain for this region + GrowableArray to_be_retained(10); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + while (_strong_code_roots_list->is_nonempty()) { + nmethod *nm = _strong_code_roots_list->pop(); + if (nm != NULL) { + NMethodMigrationOopClosure oop_cl(g1h, hr(), nm); + nm->oops_do(&oop_cl); + if (oop_cl.retain()) { + to_be_retained.push(nm); + } + } + } + + // Now push any code roots we need to retain + // FIXME: assert that region got an evacuation failure if non-empty + while (to_be_retained.is_nonempty()) { + nmethod* nm = to_be_retained.pop(); + assert(nm != NULL, "sanity"); + add_strong_code_root(nm); + } +} + +void HeapRegionRemSet::strong_code_roots_do(CodeBlobClosure* blk) const { + for (int i = 0; i < _strong_code_roots_list->length(); i += 1) { + nmethod* nm = _strong_code_roots_list->at(i); + blk->do_code_blob(nm); + } +} + +size_t HeapRegionRemSet::strong_code_roots_mem_size() { + return sizeof(GrowableArray) + + _strong_code_roots_list->max_length() * sizeof(nmethod*); +} + //-------------------- Iteration -------------------- HeapRegionRemSetIterator:: HeapRegionRemSetIterator(const HeapRegionRemSet* hrrs) : --- old/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp 2013-06-26 14:24:06.186829719 -0700 +++ new/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp 2013-06-26 14:24:05.972313883 -0700 @@ -37,6 +37,7 @@ class HeapRegionRemSetIterator; class PerRegionTable; class SparsePRT; +class nmethod; // Essentially a wrapper around SparsePRTCleanupTask. See // sparsePRT.hpp for more details. @@ -191,6 +192,10 @@ G1BlockOffsetSharedArray* _bosa; G1BlockOffsetSharedArray* bosa() const { return _bosa; } + // A list of code blobs (nmethods) whose code contains pointers into + // the region that owns this RSet. + GrowableArray* _strong_code_roots_list; + OtherRegionsTable _other_regions; enum ParIterState { Unclaimed, Claimed, Complete }; @@ -282,11 +287,13 @@ } // The actual # of bytes this hr_remset takes up. + // Note also includes the strong code root set. size_t mem_size() { return _other_regions.mem_size() // This correction is necessary because the above includes the second // part. - + sizeof(this) - sizeof(OtherRegionsTable); + + (sizeof(this) - sizeof(OtherRegionsTable)) + + strong_code_roots_mem_size(); } // Returns the memory occupancy of all static data structures associated @@ -304,6 +311,37 @@ bool contains_reference(OopOrNarrowOopStar from) const { return _other_regions.contains_reference(from); } + + // Routines for managing the list of code roots that point into + // the heap region that owns this RSet. + void add_strong_code_root(nmethod* nm); + void remove_strong_code_root(nmethod* nm); + + // During a collection, migrate the successfully evacuated strong + // code roots that referenced into the region that owns this RSet + // to the RSets of the new regions that they now point into. + // Unsuccessfully evacuated code roots are not migrated. + void migrate_strong_code_roots(); + + // Applies blk->do_code_blob() to each of the entries in + // the strong code roots list + void strong_code_roots_do(CodeBlobClosure* blk) const; + + // Returns the number of elements in the strong code roots list + int strong_code_roots_list_length() { + return _strong_code_roots_list->length(); + } + + // Returns true if the strong code roots contains the given + // nmethod. + bool strong_code_roots_list_contains(nmethod* nm) { + return _strong_code_roots_list->contains(nm); + } + + // Returns the amount of memory, in bytes, currently + // consumed by the strong code roots. + size_t strong_code_roots_mem_size(); + void print() const; // Called during a stop-world phase to perform any deferred cleanups. --- old/src/share/vm/utilities/growableArray.hpp 2013-06-26 14:24:07.621467615 -0700 +++ new/src/share/vm/utilities/growableArray.hpp 2013-06-26 14:24:07.394326355 -0700 @@ -194,6 +194,7 @@ void clear() { _len = 0; } int length() const { return _len; } + int max_length() const { return _max; } void trunc_to(int l) { assert(l <= _len,"cannot increase length"); _len = l; } bool is_empty() const { return _len == 0; } bool is_nonempty() const { return _len != 0; }