--- old/src/share/vm/c1/c1_Runtime1.cpp 2013-06-26 14:23:11.964656167 -0700 +++ new/src/share/vm/c1/c1_Runtime1.cpp 2013-06-26 14:23:11.681037892 -0700 @@ -914,16 +914,6 @@ // Return to the now deoptimized frame. } - // If we are patching in a non-perm oop, make sure the nmethod - // is on the right list. - if (ScavengeRootsInCode && mirror.not_null() && mirror()->is_scavengable()) { - MutexLockerEx ml_code (CodeCache_lock, Mutex::_no_safepoint_check_flag); - nmethod* nm = CodeCache::find_nmethod(caller_frame.pc()); - guarantee(nm != NULL, "only nmethods can contain non-perm oops"); - if (!nm->on_scavenge_root_list()) - CodeCache::add_scavenge_root_nmethod(nm); - } - // Now copy code back { @@ -1124,6 +1114,21 @@ } } } + + // If we are patching in a non-perm oop, make sure the nmethod + // is on the right list. + if (ScavengeRootsInCode && mirror.not_null() && mirror()->is_scavengable()) { + MutexLockerEx ml_code (CodeCache_lock, Mutex::_no_safepoint_check_flag); + nmethod* nm = CodeCache::find_nmethod(caller_frame.pc()); + guarantee(nm != NULL, "only nmethods can contain non-perm oops"); + if (!nm->on_scavenge_root_list()) { + CodeCache::add_scavenge_root_nmethod(nm); + } + + // Since we've patched some oops in the nmethod, + // (re)register it with the heap. + Universe::heap()->register_nmethod(nm); + } JRT_END // --- old/src/share/vm/code/nmethod.cpp 2013-06-26 14:23:13.954536561 -0700 +++ new/src/share/vm/code/nmethod.cpp 2013-06-26 14:23:13.690650019 -0700 @@ -687,6 +687,7 @@ code_buffer->copy_values_to(this); if (ScavengeRootsInCode && detect_scavenge_root_oops()) { CodeCache::add_scavenge_root_nmethod(this); + Universe::heap()->register_nmethod(this); } debug_only(verify_scavenge_root_oops()); CodeCache::commit(this); @@ -881,6 +882,7 @@ dependencies->copy_to(this); if (ScavengeRootsInCode && detect_scavenge_root_oops()) { CodeCache::add_scavenge_root_nmethod(this); + Universe::heap()->register_nmethod(this); } debug_only(verify_scavenge_root_oops()); @@ -1337,6 +1339,12 @@ inc_decompile_count(); } + // If the state is becoming a zombie, unregister the nmethod with heap + // This nmethod may have already been unloaded during a full GC. + if ((state == zombie) && !is_unloaded()) { + Universe::heap()->unregister_nmethod(this); + } + // Change state _state = state; --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2013-06-26 14:23:16.774880447 -0700 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2013-06-26 14:23:15.758780141 -0700 @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "code/codeCache.hpp" #include "code/icBuffer.hpp" #include "gc_implementation/g1/bufferingOopClosure.hpp" #include "gc_implementation/g1/concurrentG1Refine.hpp" @@ -1176,6 +1177,8 @@ bool doHeapRegion(HeapRegion* r) { if (r->continuesHumongous()) { + // We'll assert that the strong code root list is empty + assert(r->strong_code_root_list()->is_empty(), "sanity"); return false; } _g1h->reset_gc_time_stamps(r); @@ -1187,6 +1190,10 @@ // from being enqueued, and cause it to be missed. // Re: the performance cost: we shouldn't be doing full GC anyway! _mr_bs->clear(MemRegion(r->bottom(), r->end())); + + // We'll also clear the strong code root list for this region + r->strong_code_root_list()->clear(); + return false; } }; @@ -1470,6 +1477,9 @@ heap_region_iterate(&rebuild_rs); } + // Rebuild the strong code root lists for each region + rebuild_strong_code_roots(); + if (true) { // FIXME MetaspaceGC::compute_new_size(); } @@ -3066,6 +3076,76 @@ void do_oop(narrowOop* p) { do_oop_nv(p); } }; +class G1VerifyCodeRootOopClosure: public OopsInGenClosure { + G1CollectedHeap* _g1h; + OopClosure* _root_cl; + nmethod* _nm; + VerifyOption _vo; + bool _failures; + + template void do_oop_work(T* p) { + // First verify that this root is live + _root_cl->do_oop(p); + + if (!G1VerifyHeapRegionCodeRoots) { + // We're not verifying the code roots attached to heap region. + return; + } + + // Don't check the code roots during marking verification in a full GC + if (_vo == VerifyOption_G1UseMarkWord) { + return; + } + + // Now verify that the current nmethod (which contains p) is + // in the code root list of the heap region containing the + // object referenced by p. + + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + + // Now fetch the region containing the object + HeapRegion* hr = _g1h->heap_region_containing(obj); + // Verify that the strong code root list for this region + // contains the nmethod + if (!hr->strong_code_root_list()->contains(_nm)) { + gclog_or_tty->print_cr("Code root location "PTR_FORMAT" " + "from nmethod "PTR_FORMAT" not in strong " + "code roots for region ["PTR_FORMAT","PTR_FORMAT")", + p, _nm, hr->bottom(), hr->end()); + _failures = true; + } + } + } + +public: + G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo): + _g1h(g1h), _root_cl(root_cl), _vo(vo), _nm(NULL), _failures(false) {} + + void do_oop(oop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + + void set_nmethod(nmethod* nm) { _nm = nm; } + bool failures() { return _failures; } +}; + +class G1VerifyCodeRootBlobClosure: public CodeBlobClosure { + G1VerifyCodeRootOopClosure* _oop_cl; + +public: + G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl): + _oop_cl(oop_cl) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = cb->as_nmethod_or_null(); + if (nm != NULL) { + _oop_cl->set_nmethod(nm); + nm->oops_do(_oop_cl); + } + } +}; + class YoungRefCounterClosure : public OopClosure { G1CollectedHeap* _g1h; int _count; @@ -3268,7 +3348,8 @@ if (!silent) { gclog_or_tty->print("Roots "); } VerifyRootsClosure rootsCl(vo); - CodeBlobToOopClosure blobsCl(&rootsCl, /*do_marking=*/ false); + G1VerifyCodeRootOopClosure codeRootsCl(this, &rootsCl, vo); + G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl); VerifyKlassClosure klassCl(this, &rootsCl); // We apply the relevant closures to all the oops in the @@ -3287,7 +3368,7 @@ &klassCl ); - bool failures = rootsCl.failures(); + bool failures = rootsCl.failures() || codeRootsCl.failures(); if (vo != VerifyOption_G1UseMarkWord) { // If we're verifying during a full GC then the region sets @@ -3796,8 +3877,9 @@ append_secondary_free_list_if_not_empty_with_lock(); } - assert(check_young_list_well_formed(), - "young list should be well formed"); + assert(check_young_list_well_formed(), "young list should be well formed"); + assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue), + "sanity check"); // Don't dynamically change the number of GC threads this early. A value of // 0 is used to indicate serial work. When parallel work is done, @@ -4901,7 +4983,11 @@ G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, &pss); - int so = SharedHeap::SO_AllClasses | SharedHeap::SO_Strings | SharedHeap::SO_CodeCache; + // Don't scan the scavengable methods in the code cache as part + // of strong root scanning. The code roots that point into a + // region in the collection set are scanned when we scan the + // region's RSet. + int so = SharedHeap::SO_AllClasses | SharedHeap::SO_Strings; pss.start_strong_roots(); _g1h->g1_process_strong_roots(/* is scavenging */ true, @@ -4943,67 +5029,6 @@ // *** Common G1 Evacuation Stuff -// Closures that support the filtering of CodeBlobs scanned during -// external root scanning. - -// Closure applied to reference fields in code blobs (specifically nmethods) -// to determine whether an nmethod contains references that point into -// the collection set. Used as a predicate when walking code roots so -// that only nmethods that point into the collection set are added to the -// 'marked' list. - -class G1FilteredCodeBlobToOopClosure : public CodeBlobToOopClosure { - - class G1PointsIntoCSOopClosure : public OopClosure { - G1CollectedHeap* _g1; - bool _points_into_cs; - public: - G1PointsIntoCSOopClosure(G1CollectedHeap* g1) : - _g1(g1), _points_into_cs(false) { } - - bool points_into_cs() const { return _points_into_cs; } - - template - void do_oop_nv(T* p) { - if (!_points_into_cs) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop) && - _g1->in_cset_fast_test(oopDesc::decode_heap_oop_not_null(heap_oop))) { - _points_into_cs = true; - } - } - } - - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - }; - - G1CollectedHeap* _g1; - -public: - G1FilteredCodeBlobToOopClosure(G1CollectedHeap* g1, OopClosure* cl) : - CodeBlobToOopClosure(cl, true), _g1(g1) { } - - virtual void do_code_blob(CodeBlob* cb) { - nmethod* nm = cb->as_nmethod_or_null(); - if (nm != NULL && !(nm->test_oops_do_mark())) { - G1PointsIntoCSOopClosure predicate_cl(_g1); - nm->oops_do(&predicate_cl); - - if (predicate_cl.points_into_cs()) { - // At least one of the reference fields or the oop relocations - // in the nmethod points into the collection set. We have to - // 'mark' this nmethod. - // Note: Revisit the following if CodeBlobToOopClosure::do_code_blob() - // or MarkingCodeBlobClosure::do_code_blob() change. - if (!nm->test_set_oops_do_mark()) { - do_newly_marked_nmethod(nm); - } - } - } - } -}; - // This method is run in a GC worker. void @@ -5021,9 +5046,10 @@ BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots); - // Walk the code cache w/o buffering, because StarTask cannot handle - // unaligned oop locations. - G1FilteredCodeBlobToOopClosure eager_scan_code_roots(this, scan_non_heap_roots); + assert(so & SO_CodeCache || scan_rs != NULL, "must scan code roots somehow"); + // Walk the code cache/strong code roots w/o buffering, because StarTask + // cannot handle unaligned oop locations. + CodeBlobToOopClosure eager_scan_code_roots(scan_non_heap_roots, true /* do_marking */); process_strong_roots(false, // no scoping; this is parallel code is_scavenging, so, @@ -5068,9 +5094,22 @@ } g1_policy()->phase_times()->record_satb_filtering_time(worker_i, satb_filtering_ms); + // If this is an initial mark pause, and we're not scanning + // the entire code cache, we need to mark the oops in the + // strong code root lists for the regions that are not in + // the collection set. + // Note all threads participate in this set of root tasks. + double mark_strong_code_roots_ms = 0.0; + if (g1_policy()->during_initial_mark_pause() && !(so & SO_CodeCache)) { + double mark_strong_roots_start = os::elapsedTime(); + mark_strong_code_roots(worker_i); + mark_strong_code_roots_ms = (os::elapsedTime() - mark_strong_roots_start) * 1000.0; + } + g1_policy()->phase_times()->record_strong_code_root_mark_time(worker_i, mark_strong_code_roots_ms); + // Now scan the complement of the collection set. if (scan_rs != NULL) { - g1_rem_set()->oops_into_collection_set_do(scan_rs, worker_i); + g1_rem_set()->oops_into_collection_set_do(scan_rs, &eager_scan_code_roots, worker_i); } _process_strong_tasks->all_tasks_completed(); } @@ -5696,6 +5735,17 @@ hot_card_cache->reset_hot_cache(); hot_card_cache->set_use_cache(true); + // Migrate the strong code roots attached to each region in + // the collection set. Ideally we would like to do this + // after we have finished the scanning/evacuation of the + // strong code roots for a particular heap region. + migrate_strong_code_roots(); + + if (g1_policy()->during_initial_mark_pause()) { + // Reset the claim values set during marking the strong code roots + reset_heap_region_claim_values(); + } + finalize_for_evac_failure(); if (evacuation_failed()) { @@ -6490,3 +6540,199 @@ _humongous_set.verify_end(); _free_list.verify_end(); } + +// Optimized nmethod scanning + +class RegisterNMethodOopClosure: public OopClosure { + G1CollectedHeap* _g1h; + nmethod* _nm; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + HeapRegion* hr = _g1h->heap_region_containing(obj); + assert(!hr->isHumongous(), "code root in humongous region?"); + + // HeapRegion::add_strong_code_root() avoids adding duplicate + // entries but having duplicates is OK since we "mark" nmethods + // as visited when we scan the strong code root lists during the GC. + hr->add_strong_code_root(_nm); + assert(hr->strong_code_root_list()->contains(_nm), "push failed?"); + } + } + +public: + RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) : + _g1h(g1h), _nm(nm) {} + + void do_oop(oop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } +}; + +class UnregisterNMethodOopClosure: public OopClosure { + G1CollectedHeap* _g1h; + nmethod* _nm; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + HeapRegion* hr = _g1h->heap_region_containing(obj); + assert(!hr->isHumongous(), "code root in humongous region?"); + hr->remove_strong_code_root(_nm); + assert(!hr->strong_code_root_list()->contains(_nm), "remove failed?"); + } + } + +public: + UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) : + _g1h(g1h), _nm(nm) {} + + void do_oop(oop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } +}; + +void G1CollectedHeap::register_nmethod(nmethod* nm) { + guarantee(nm != NULL, "sanity"); + RegisterNMethodOopClosure reg_cl(this, nm); + nm->oops_do(®_cl); +} + +void G1CollectedHeap::unregister_nmethod(nmethod* nm) { + guarantee(nm != NULL, "sanity"); + UnregisterNMethodOopClosure reg_cl(this, nm); + nm->oops_do(®_cl); +} + +class MigrateCodeRootsHeapRegionClosure: public HeapRegionClosure { +public: + bool doHeapRegion(HeapRegion *hr) { + assert(!hr->isHumongous(), "humongous region in collection set?"); + hr->migrate_strong_code_roots(); + return false; + } +}; + +void G1CollectedHeap::migrate_strong_code_roots() { + MigrateCodeRootsHeapRegionClosure cl; + double migrate_start = os::elapsedTime(); + collection_set_iterate(&cl); + double migration_time_ms = (os::elapsedTime() - migrate_start) * 1000.0; + g1_policy()->phase_times()->record_strong_code_root_migration_time(migration_time_ms); +} + +// Mark all the code roots that point into regions *not* in the +// collection set. +// +// Note we do not want to use a "marking" CodeBlobToOopClosure while +// walking the the code roots lists of regions not in the collection +// set. Suppose we have an nmethod (M) that points to objects in two +// separate regions - one in the collection set (R1) and one not (R2). +// Using a "marking" CodeBlobToOopClosure here would result in "marking" +// nmethod M when walking the code roots for R1. When we come to scan +// the code roots for R2, we would see that M is already marked and it +// would be skipped and the objects in R2 that are referenced from M +// would not be evacuated. + +class MarkStrongCodeRootCodeBlobClosure: public CodeBlobClosure { + + class MarkStrongCodeRootOopClosure: public OopClosure { + ConcurrentMark* _cm; + HeapRegion* _hr; + uint _worker_id; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + // Only mark objects in the region (which is assumed + // to be not in the collection set). + if (_hr->is_in(obj)) { + _cm->grayRoot(obj, (size_t) obj->size(), _worker_id); + } + } + } + + public: + MarkStrongCodeRootOopClosure(ConcurrentMark* cm, HeapRegion* hr, uint worker_id) : + _cm(cm), _hr(hr), _worker_id(worker_id) { + assert(!_hr->in_collection_set(), "sanity"); + } + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } + }; + + MarkStrongCodeRootOopClosure _oop_cl; + +public: + MarkStrongCodeRootCodeBlobClosure(ConcurrentMark* cm, HeapRegion* hr, uint worker_id): + _oop_cl(cm, hr, worker_id) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null(); + if (nm != NULL) { + nm->oops_do(&_oop_cl); + } + } +}; + +class MarkStrongCodeRootsHRClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + uint _worker_id; + +public: + MarkStrongCodeRootsHRClosure(G1CollectedHeap* g1h, uint worker_id) : + _g1h(g1h), _worker_id(worker_id) {} + + bool doHeapRegion(HeapRegion *hr) { + if (hr->isHumongous()) { + // Code roots should never be attached to a humongous region + assert(hr->strong_code_root_list()->is_empty(), "sanity"); + return false; + } + + if (hr->in_collection_set()) { + // Don't mark code roots into regions in the collection set here. + // They will be marked when we scan them. + return false; + } + + MarkStrongCodeRootCodeBlobClosure cb_cl(_g1h->concurrent_mark(), hr, _worker_id); + hr->strong_code_roots_do(&cb_cl); + return false; + } +}; + +void G1CollectedHeap::mark_strong_code_roots(uint worker_id) { + MarkStrongCodeRootsHRClosure cl(this, worker_id); + heap_region_par_iterate_chunked(&cl, + worker_id, + workers()->active_workers(), + HeapRegion::ParMarkRootClaimValue); +} + +class RebuildStrongCodeRootClosure: public CodeBlobClosure { + G1CollectedHeap* _g1h; + +public: + RebuildStrongCodeRootClosure(G1CollectedHeap* g1h) : + _g1h(g1h) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = (cb != NULL) ? cb->as_nmethod_or_null() : NULL; + if (nm == NULL) { + return; + } + + if (ScavengeRootsInCode && nm->detect_scavenge_root_oops()) { + _g1h->register_nmethod(nm); + } + } +}; + +void G1CollectedHeap::rebuild_strong_code_roots() { + RebuildStrongCodeRootClosure blob_cl(this); + CodeCache::blobs_do(&blob_cl); +} --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2013-06-26 14:23:19.263889943 -0700 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2013-06-26 14:23:18.908048901 -0700 @@ -63,6 +63,7 @@ class ConcurrentMarkThread; class ConcurrentG1Refine; class GenerationCounters; +class nmethod; typedef OverflowTaskQueue RefToScanQueue; typedef GenericTaskQueueSet RefToScanQueueSet; @@ -1629,6 +1630,28 @@ void push_dirty_cards_region(HeapRegion* hr); HeapRegion* pop_dirty_cards_region(); + // Optimized nmethod scanning support routines + + // Register the given nmethod with the G1 heap + virtual void register_nmethod(nmethod* nm); + + // Unregister the given nmethod from the G1 heap + virtual void unregister_nmethod(nmethod* nm); + + // Migrate the nmethods in the code root lists of the regions + // in the collection set to regions in to-space. In the event + // of an evacuation failure, nmethods that reference objects + // that were not successfullly evacuated are not migrated. + void migrate_strong_code_roots(); + + // During an initial mark pause, mark all the code roots that + // point into regions *not* in the collection set. + void mark_strong_code_roots(uint worker_id); + + // Rebuild the stong code root lists for each region + // after a full GC + void rebuild_strong_code_roots(); + // Verification // The following is just to alert the verification code --- old/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp 2013-06-26 14:23:21.026695356 -0700 +++ new/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp 2013-06-26 14:23:20.478780483 -0700 @@ -161,6 +161,8 @@ _last_update_rs_times_ms(_max_gc_threads, "%.1lf"), _last_update_rs_processed_buffers(_max_gc_threads, "%d"), _last_scan_rs_times_ms(_max_gc_threads, "%.1lf"), + _last_strong_code_root_scan_times_ms(_max_gc_threads, "%.1lf"), + _last_strong_code_root_mark_times_ms(_max_gc_threads, "%.1lf"), _last_obj_copy_times_ms(_max_gc_threads, "%.1lf"), _last_termination_times_ms(_max_gc_threads, "%.1lf"), _last_termination_attempts(_max_gc_threads, SIZE_FORMAT), @@ -182,6 +184,8 @@ _last_update_rs_times_ms.reset(); _last_update_rs_processed_buffers.reset(); _last_scan_rs_times_ms.reset(); + _last_strong_code_root_scan_times_ms.reset(); + _last_strong_code_root_mark_times_ms.reset(); _last_obj_copy_times_ms.reset(); _last_termination_times_ms.reset(); _last_termination_attempts.reset(); @@ -197,6 +201,8 @@ _last_update_rs_times_ms.verify(); _last_update_rs_processed_buffers.verify(); _last_scan_rs_times_ms.verify(); + _last_strong_code_root_scan_times_ms.verify(); + _last_strong_code_root_mark_times_ms.verify(); _last_obj_copy_times_ms.verify(); _last_termination_times_ms.verify(); _last_termination_attempts.verify(); @@ -210,6 +216,8 @@ _last_satb_filtering_times_ms.get(i) + _last_update_rs_times_ms.get(i) + _last_scan_rs_times_ms.get(i) + + _last_strong_code_root_scan_times_ms.get(i) + + _last_strong_code_root_mark_times_ms.get(i) + _last_obj_copy_times_ms.get(i) + _last_termination_times_ms.get(i); @@ -239,6 +247,9 @@ // Now subtract the time taken to fix up roots in generated code misc_time_ms += _cur_collection_code_root_fixup_time_ms; + // Strong code root migration time + misc_time_ms += _cur_strong_code_root_migration_time_ms; + // Subtract the time taken to clean the card table from the // current value of "other time" misc_time_ms += _cur_clear_ct_time_ms; @@ -257,9 +268,13 @@ if (_last_satb_filtering_times_ms.sum() > 0.0) { _last_satb_filtering_times_ms.print(2, "SATB Filtering (ms)"); } + if (_last_strong_code_root_mark_times_ms.sum() > 0.0) { + _last_strong_code_root_mark_times_ms.print(2, "Code Root Marking (ms)"); + } _last_update_rs_times_ms.print(2, "Update RS (ms)"); _last_update_rs_processed_buffers.print(3, "Processed Buffers"); _last_scan_rs_times_ms.print(2, "Scan RS (ms)"); + _last_strong_code_root_scan_times_ms.print(2, "Code Root Scanning (ms)"); _last_obj_copy_times_ms.print(2, "Object Copy (ms)"); _last_termination_times_ms.print(2, "Termination (ms)"); if (G1Log::finest()) { @@ -273,12 +288,17 @@ if (_last_satb_filtering_times_ms.sum() > 0.0) { _last_satb_filtering_times_ms.print(1, "SATB Filtering (ms)"); } + if (_last_strong_code_root_mark_times_ms.sum() > 0.0) { + _last_strong_code_root_mark_times_ms.print(1, "Code Root Marking (ms)"); + } _last_update_rs_times_ms.print(1, "Update RS (ms)"); _last_update_rs_processed_buffers.print(2, "Processed Buffers"); _last_scan_rs_times_ms.print(1, "Scan RS (ms)"); + _last_strong_code_root_scan_times_ms.print(1, "Code Root Scanning (ms)"); _last_obj_copy_times_ms.print(1, "Object Copy (ms)"); } print_stats(1, "Code Root Fixup", _cur_collection_code_root_fixup_time_ms); + print_stats(1, "Code Root Migration", _cur_strong_code_root_migration_time_ms); print_stats(1, "Clear CT", _cur_clear_ct_time_ms); double misc_time_ms = pause_time_sec * MILLIUNITS - accounted_time_ms(); print_stats(1, "Other", misc_time_ms); --- old/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp 2013-06-26 14:23:24.455920890 -0700 +++ new/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp 2013-06-26 14:23:24.245293436 -0700 @@ -119,6 +119,8 @@ WorkerDataArray _last_update_rs_times_ms; WorkerDataArray _last_update_rs_processed_buffers; WorkerDataArray _last_scan_rs_times_ms; + WorkerDataArray _last_strong_code_root_scan_times_ms; + WorkerDataArray _last_strong_code_root_mark_times_ms; WorkerDataArray _last_obj_copy_times_ms; WorkerDataArray _last_termination_times_ms; WorkerDataArray _last_termination_attempts; @@ -128,6 +130,7 @@ double _cur_collection_par_time_ms; double _cur_collection_code_root_fixup_time_ms; + double _cur_strong_code_root_migration_time_ms; double _cur_clear_ct_time_ms; double _cur_ref_proc_time_ms; @@ -179,6 +182,14 @@ _last_scan_rs_times_ms.set(worker_i, ms); } + void record_strong_code_root_scan_time(uint worker_i, double ms) { + _last_strong_code_root_scan_times_ms.set(worker_i, ms); + } + + void record_strong_code_root_mark_time(uint worker_i, double ms) { + _last_strong_code_root_mark_times_ms.set(worker_i, ms); + } + void record_obj_copy_time(uint worker_i, double ms) { _last_obj_copy_times_ms.set(worker_i, ms); } @@ -208,6 +219,10 @@ _cur_collection_code_root_fixup_time_ms = ms; } + void record_strong_code_root_migration_time(double ms) { + _cur_strong_code_root_migration_time_ms = ms; + } + void record_ref_proc_time(double ms) { _cur_ref_proc_time_ms = ms; } @@ -294,6 +309,14 @@ return _last_scan_rs_times_ms.average(); } + double average_last_strong_code_root_scan_time(){ + return _last_strong_code_root_scan_times_ms.average(); + } + + double average_last_strong_code_root_mark_time(){ + return _last_strong_code_root_mark_times_ms.average(); + } + double average_last_obj_copy_time() { return _last_obj_copy_times_ms.average(); } --- old/src/share/vm/gc_implementation/g1/g1RemSet.cpp 2013-06-26 14:23:26.325357256 -0700 +++ new/src/share/vm/gc_implementation/g1/g1RemSet.cpp 2013-06-26 14:23:25.896554484 -0700 @@ -104,15 +104,25 @@ class ScanRSClosure : public HeapRegionClosure { size_t _cards_done, _cards; G1CollectedHeap* _g1h; + OopsInHeapRegionClosure* _oc; + CodeBlobToOopClosure* _code_root_cl; + G1BlockOffsetSharedArray* _bot_shared; CardTableModRefBS *_ct_bs; - int _worker_i; - int _block_size; - bool _try_claimed; + + double _strong_code_root_scan_time_sec; + int _worker_i; + int _block_size; + bool _try_claimed; + public: - ScanRSClosure(OopsInHeapRegionClosure* oc, int worker_i) : + ScanRSClosure(OopsInHeapRegionClosure* oc, + CodeBlobToOopClosure* code_root_cl, + int worker_i) : _oc(oc), + _code_root_cl(code_root_cl), + _strong_code_root_scan_time_sec(0.0), _cards(0), _cards_done(0), _worker_i(worker_i), @@ -160,6 +170,12 @@ card_start, card_start + G1BlockOffsetSharedArray::N_words); } + void scan_strong_code_roots(HeapRegion* r) { + double scan_start = os::elapsedTime(); + r->strong_code_roots_do(_code_root_cl); + _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start); + } + bool doHeapRegion(HeapRegion* r) { assert(r->in_collection_set(), "should only be called on elements of CS."); HeapRegionRemSet* hrrs = r->rem_set(); @@ -173,6 +189,7 @@ // _try_claimed || r->claim_iter() // is true: either we're supposed to work on claimed-but-not-complete // regions, or we successfully claimed the region. + HeapRegionRemSetIterator iter(hrrs); size_t card_index; @@ -205,30 +222,43 @@ } } if (!_try_claimed) { + // Scan the strong code root list attached to the current region + scan_strong_code_roots(r); + hrrs->set_iter_complete(); } return false; } + + double strong_code_root_scan_time_sec() { + return _strong_code_root_scan_time_sec; + } + size_t cards_done() { return _cards_done;} size_t cards_looked_up() { return _cards;} }; -void G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) { +void G1RemSet::scanRS(OopsInHeapRegionClosure* oc, + CodeBlobToOopClosure* code_root_cl, + int worker_i) { double rs_time_start = os::elapsedTime(); HeapRegion *startRegion = _g1->start_cset_region_for_worker(worker_i); - ScanRSClosure scanRScl(oc, worker_i); + ScanRSClosure scanRScl(oc, code_root_cl, worker_i); _g1->collection_set_iterate_from(startRegion, &scanRScl); scanRScl.set_try_claimed(); _g1->collection_set_iterate_from(startRegion, &scanRScl); - double scan_rs_time_sec = os::elapsedTime() - rs_time_start; + double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) + - scanRScl.strong_code_root_scan_time_sec(); - assert( _cards_scanned != NULL, "invariant" ); + assert(_cards_scanned != NULL, "invariant"); _cards_scanned[worker_i] = scanRScl.cards_done(); _g1p->phase_times()->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0); + _g1p->phase_times()->record_strong_code_root_scan_time(worker_i, + scanRScl.strong_code_root_scan_time_sec() * 1000.0); } // Closure used for updating RSets and recording references that @@ -288,7 +318,8 @@ } void G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc, - int worker_i) { + CodeBlobToOopClosure* code_root_cl, + int worker_i) { #if CARD_REPEAT_HISTO ct_freq_update_histo_and_reset(); #endif @@ -328,7 +359,7 @@ _g1p->phase_times()->record_update_rs_time(worker_i, 0.0); } if (G1UseParallelRSetScanning || (worker_i == 0)) { - scanRS(oc, worker_i); + scanRS(oc, code_root_cl, worker_i); } else { _g1p->phase_times()->record_scan_rs_time(worker_i, 0.0); } --- old/src/share/vm/gc_implementation/g1/g1RemSet.hpp 2013-06-26 14:23:28.109622017 -0700 +++ new/src/share/vm/gc_implementation/g1/g1RemSet.hpp 2013-06-26 14:23:27.869760177 -0700 @@ -81,14 +81,23 @@ G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs); ~G1RemSet(); - // Invoke "blk->do_oop" on all pointers into the CS in objects in regions - // outside the CS (having invoked "blk->set_region" to set the "from" - // region correctly beforehand.) The "worker_i" param is for the - // parallel case where the number of the worker thread calling this - // function can be helpful in partitioning the work to be done. It - // should be the same as the "i" passed to the calling thread's - // work(i) function. In the sequential case this param will be ingored. - void oops_into_collection_set_do(OopsInHeapRegionClosure* blk, int worker_i); + // Invoke "blk->do_oop" on all pointers into the collection set + // from objects in regions outside the collection set (having + // invoked "blk->set_region" to set the "from" region correctly + // beforehand.) + // + // Invoke code_root_cl->do_code_blob on the unmarked nmethods + // on the strong code roots list for each region in the + // collection set. + // + // The "worker_i" param is for the parallel case where the id + // of the worker thread calling this function can be helpful in + // partitioning the work to be done. It should be the same as + // the "i" passed to the calling thread's work(i) function. + // In the sequential case this param will be ignored. + void oops_into_collection_set_do(OopsInHeapRegionClosure* blk, + CodeBlobToOopClosure* code_root_cl, + int worker_i); // Prepare for and cleanup after an oops_into_collection_set_do // call. Must call each of these once before and after (in sequential @@ -98,7 +107,10 @@ void prepare_for_oops_into_collection_set_do(); void cleanup_after_oops_into_collection_set_do(); - void scanRS(OopsInHeapRegionClosure* oc, int worker_i); + void scanRS(OopsInHeapRegionClosure* oc, + CodeBlobToOopClosure* code_root_cl, + int worker_i); + void updateRS(DirtyCardQueue* into_cset_dcq, int worker_i); CardTableModRefBS* ct_bs() { return _ct_bs; } --- old/src/share/vm/gc_implementation/g1/g1_globals.hpp 2013-06-26 14:23:30.101065060 -0700 +++ new/src/share/vm/gc_implementation/g1/g1_globals.hpp 2013-06-26 14:23:29.886650279 -0700 @@ -333,7 +333,10 @@ \ diagnostic(bool, G1VerifyRSetsDuringFullGC, false, \ "If true, perform verification of each heap region's " \ - "remembered set when verifying the heap during a full GC.") + "remembered set when verifying the heap during a full GC.") \ + \ + diagnostic(bool, G1VerifyHeapRegionCodeRoots, false, \ + "Verify the code root lists attached to each heap region.") G1_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG, DECLARE_MANAGEABLE_FLAG, DECLARE_PRODUCT_RW_FLAG) --- old/src/share/vm/gc_implementation/g1/heapRegion.cpp 2013-06-26 14:23:31.621634078 -0700 +++ new/src/share/vm/gc_implementation/g1/heapRegion.cpp 2013-06-26 14:23:31.395569667 -0700 @@ -32,6 +32,7 @@ #include "memory/genOopClosures.inline.hpp" #include "memory/iterator.hpp" #include "oops/oop.inline.hpp" +#include "utilities/growableArray.hpp" int HeapRegion::LogOfHRGrainBytes = 0; int HeapRegion::LogOfHRGrainWords = 0; @@ -232,6 +233,13 @@ _offsets.resize(HeapRegion::GrainWords); init_top_at_mark_start(); + + if (_strong_code_root_list != NULL) { + delete _strong_code_root_list; + } + _strong_code_root_list = new (ResourceObj::C_HEAP, mtGC) + GrowableArray(10, 0, NULL, true); + if (clear_space) clear(SpaceDecorator::Mangle); } @@ -360,7 +368,7 @@ #endif // ASSERT _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0), - _predicted_bytes_to_copy(0) + _predicted_bytes_to_copy(0), _strong_code_root_list(NULL) { _orig_end = mr.end(); // Note that initialize() will set the start of the unmarked area of the @@ -590,6 +598,242 @@ return NULL; } +// Code roots support + +void HeapRegion::add_strong_code_root(nmethod* nm) { + assert(nm != NULL, "sanity"); + // Search for the code blob from the RHS to avoid + // duplicate entries as much as possible + if (_strong_code_root_list->find_from_end(nm) < 0) { + // Code blob isn't already in the list + _strong_code_root_list->push(nm); + } +} + +void HeapRegion::remove_strong_code_root(nmethod* nm) { + assert(nm != NULL, "sanity"); + int idx = _strong_code_root_list->find(nm); + while (idx >= 0) { + _strong_code_root_list->remove_at(idx); + idx = _strong_code_root_list->find(nm); + } +} + +class NMethodMigrationOopClosure : public OopClosure { + G1CollectedHeap* _g1h; + HeapRegion* _from; + nmethod* _nm; + + uint _num_self_forwarded; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (_from->is_in(obj)) { + // Reference still points into the source region. + // Since roots are immediately evacuated this means that + // we must have self forwarded the object + assert(obj->is_forwarded(), + err_msg("code roots should be immediately evacuated. " + "Ref: "PTR_FORMAT", " + "Obj: "PTR_FORMAT", " + "Region: "HR_FORMAT, + p, (void*) obj, HR_FORMAT_PARAMS(_from))); + assert(obj->forwardee() == obj, + err_msg("not self forwarded? obj = "PTR_FORMAT, (void*)obj)); + + // The object has been self forwarded. + // Note, if we're during an initial mark pause, there is + // no need to explicitly mark object. It will be marked + // during the regular evacuation failure handling code. + _num_self_forwarded++; + } else { + // The reference points into a promotion or to-space region + HeapRegion* to = _g1h->heap_region_containing(obj); + to->add_strong_code_root(_nm); + } + } + } + +public: + NMethodMigrationOopClosure(G1CollectedHeap* g1h, HeapRegion* from, nmethod* nm): + _g1h(g1h), _from(from), _nm(nm), _num_self_forwarded(0) {} + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } + + uint retain() { return _num_self_forwarded > 0; } +}; + +void HeapRegion::migrate_strong_code_roots() { + assert(in_collection_set(), "only collection set regions"); + assert(!isHumongous(), "not humongous regions"); + + ResourceMark rm; + + // List of code blobs to retain for this region + GrowableArray to_be_retained(10); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + while (strong_code_root_list()->is_nonempty()) { + nmethod *nm = strong_code_root_list()->pop(); + if (nm != NULL) { + NMethodMigrationOopClosure oop_cl(g1h, this, nm); + nm->oops_do(&oop_cl); + if (oop_cl.retain()) { + to_be_retained.push(nm); + } + } + } + + // Now push any code roots we need to retain + // FIXME: assert that region got an evacuation failure if non-empty + while (to_be_retained.is_nonempty()) { + nmethod* nm = to_be_retained.pop(); + assert(nm != NULL, "sanity"); + add_strong_code_root(nm); + } +} + +void HeapRegion::strong_code_roots_do(CodeBlobClosure* blk) const { + for (int i = 0; i < _strong_code_root_list->length(); i += 1) { + nmethod* nm = _strong_code_root_list->at(i); + blk->do_code_blob(nm); + } +} + +size_t HeapRegion::strong_code_root_mem_size() { + return sizeof(GrowableArray) + + _strong_code_root_list->max_length() * sizeof(nmethod*); +} + +class VerifyStrongCodeRootOopClosure: public OopClosure { + const HeapRegion* _hr; + nmethod* _nm; + bool _failures; + bool _has_oops_in_region; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + + // Note: not all the oops embedded in the nmethod are in the + // current region. We only look at those which are. + if (_hr->is_in(obj)) { + // Object is in the region. Check that its less than top + if (_hr->top() <= (HeapWord*)obj) { + // Object is above top + gclog_or_tty->print_cr("Object "PTR_FORMAT" in region " + "["PTR_FORMAT", "PTR_FORMAT") is above " + "top "PTR_FORMAT, + obj, _hr->bottom(), _hr->end(), _hr->top()); + _failures = true; + return; + } + // Nmethod has at least one oop in the current region + _has_oops_in_region = true; + } + } + } + +public: + VerifyStrongCodeRootOopClosure(const HeapRegion* hr, nmethod* nm): + _hr(hr), _failures(false), _has_oops_in_region(false) {} + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } + + bool failures() { return _failures; } + bool has_oops_in_region() { return _has_oops_in_region; } +}; + +class VerifyStrongCodeRootCodeBlobClosure: public CodeBlobClosure { + const HeapRegion* _hr; + bool _failures; +public: + VerifyStrongCodeRootCodeBlobClosure(const HeapRegion* hr) : + _hr(hr), _failures(false) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null(); + if (nm != NULL) { + // Verify that the nemthod is live + if (!nm->is_alive()) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has dead nmethod " + PTR_FORMAT" in its strong code roots", + _hr->bottom(), _hr->end(), nm); + _failures = true; + } else { + VerifyStrongCodeRootOopClosure oop_cl(_hr, nm); + nm->oops_do(&oop_cl); + if (!oop_cl.has_oops_in_region()) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has nmethod " + PTR_FORMAT" in its strong code roots " + "with no pointers into region", + _hr->bottom(), _hr->end(), nm); + _failures = true; + } else if (oop_cl.failures()) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has other " + "failures for nmethod "PTR_FORMAT, + _hr->bottom(), _hr->end(), nm); + _failures = true; + } + } + } + } + + bool failures() { return _failures; } +}; + +void HeapRegion::verify_strong_code_roots(VerifyOption vo, bool* failures) const { + if (!G1VerifyHeapRegionCodeRoots) { + // We're not verifying code roots. + return; + } + if (vo == VerifyOption_G1UseMarkWord) { + // Marking verification during a full GC is performed after class + // unloading, code cache unloading, etc so the strong code roots + // attached to each heap region are in an inconsistent state. They won't + // be consistent until the strong code roots are rebuilt after the + // actual GC. Skip verifying the strong code roots in this particular + // time. + assert(VerifyDuringGC, "only way to get here"); + return; + } + + // if this region is empty then there should be no entries + // on its strong code root list + if (is_empty()) { + if (!_strong_code_root_list->is_empty()) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is empty " + "but has "INT32_FORMAT" code root entries", + bottom(), end(), _strong_code_root_list->length()); + *failures = true; + } + return; + } + + // An H-region should have an empty strong code root list + if (isHumongous()) { + if (!_strong_code_root_list->is_empty()) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is humongous " + "but has "INT32_FORMAT" code root entries", + bottom(), end(), _strong_code_root_list->length()); + *failures = true; + } + return; + } + + VerifyStrongCodeRootCodeBlobClosure cb_cl(this); + strong_code_roots_do(&cb_cl); + + if (cb_cl.failures()) { + *failures = true; + } +} + void HeapRegion::print() const { print_on(gclog_or_tty); } void HeapRegion::print_on(outputStream* st) const { if (isHumongous()) { @@ -894,6 +1138,8 @@ *failures = true; return; } + + verify_strong_code_roots(vo, failures); } void HeapRegion::verify() const { --- old/src/share/vm/gc_implementation/g1/heapRegion.hpp 2013-06-26 14:23:33.555054820 -0700 +++ new/src/share/vm/gc_implementation/g1/heapRegion.hpp 2013-06-26 14:23:33.307922722 -0700 @@ -52,6 +52,7 @@ class HeapRegionRemSetIterator; class HeapRegion; class HeapRegionSetBase; +class nmethod; #define HR_FORMAT "%u:(%s)["PTR_FORMAT","PTR_FORMAT","PTR_FORMAT"]" #define HR_FORMAT_PARAMS(_hr_) \ @@ -337,6 +338,9 @@ // the total value for the collection set. size_t _predicted_bytes_to_copy; + // A list of code blobs (nmethods) whose code contains pointers into this region + GrowableArray* _strong_code_root_list; + public: HeapRegion(uint hrs_index, G1BlockOffsetSharedArray* sharedOffsetArray, @@ -371,7 +375,8 @@ RebuildRSClaimValue = 5, ParEvacFailureClaimValue = 6, AggregateCountClaimValue = 7, - VerifyCountClaimValue = 8 + VerifyCountClaimValue = 8, + ParMarkRootClaimValue = 9 }; inline HeapWord* par_allocate_no_bot_updates(size_t word_size) { @@ -796,6 +801,33 @@ virtual void reset_after_compaction(); + // Routines for managing the list of code roots that point into + // this heap region. + void add_strong_code_root(nmethod* nm); + void remove_strong_code_root(nmethod* nm); + + GrowableArray* strong_code_root_list() { + return _strong_code_root_list; + } + + // During a collection, migrate successfully evacuated strong + // code roots attached to this region to the new regions that + // they point into. Unsuccessfully evacuated code roots are + // not migrated. + void migrate_strong_code_roots(); + + // Applies blk->do_code_blob() to each of the entries in + // the strong code roots list; + void strong_code_roots_do(CodeBlobClosure* blk) const; + + // Returns the amount of memory, in bytes, currently + // consumed by the strong code roots. + size_t strong_code_root_mem_size(); + + // Verify that the entries on the strong code root list are live and + // include at least one pointer into this region. + void verify_strong_code_roots(VerifyOption vo, bool* failures) const; + void print() const; void print_on(outputStream* st) const; --- old/src/share/vm/gc_interface/collectedHeap.hpp 2013-06-26 14:23:35.505918829 -0700 +++ new/src/share/vm/gc_interface/collectedHeap.hpp 2013-06-26 14:23:35.288144792 -0700 @@ -43,6 +43,7 @@ class AdaptiveSizePolicy; class Thread; class CollectorPolicy; +class nmethod; class GCMessage : public FormatBuffer<1024> { public: @@ -607,6 +608,11 @@ } } + // Registering and unregistering an nmethod (compiled code) with the heap. + // Override with specific mechanism for each specialized heap type + virtual void register_nmethod(nmethod* nm) {} + virtual void unregister_nmethod(nmethod* nm) {} + // Heap verification virtual void verify(bool silent, VerifyOption option) = 0;