< prev index next >
src/share/vm/gc_implementation/shenandoah/shenandoahCodeRoots.cpp
Print this page
rev 10509 : [backport] Refactor and improve ShenandoahCodeRoots strategies
rev 10598 : [backport] Shenandoah changes to allow enabling -Wreorder
rev 10600 : [backport] Trivial enhancement to avoid costly deletion array element
rev 10614 : [backport] Replace custom asserts with shenandoah_assert_*
rev 10619 : [backport] Move ParallelCodeIterator to ShenandoahCodeRoots
*** 27,90 ****
#include "code/nmethod.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
! class ShenandoahNMethodCountOops : public OopClosure {
! public:
! size_t _non_null_oops;
! ShenandoahNMethodCountOops() : _non_null_oops(0) {};
! private:
! template <class T>
! inline void do_oop_work(T* p) {
! T o = oopDesc::load_heap_oop(p);
! if (! oopDesc::is_null(o)) {
! _non_null_oops++;
! }
}
! public:
! void do_oop(oop* o) {
! do_oop_work(o);
}
- void do_oop(narrowOop* o) {
- do_oop_work(o);
}
- bool has_oops() {
- return _non_null_oops > 0;
}
- };
! class ShenandoahNMethodHasCSetOops : public OopClosure {
! public:
! ShenandoahHeap* _heap;
! bool _has_cset_oops;
! ShenandoahNMethodHasCSetOops(ShenandoahHeap* heap) : _heap(heap), _has_cset_oops(false) {};
private:
! template <class T>
! inline void do_oop_work(T* p) {
! if (_has_cset_oops) return;
! T o = oopDesc::load_heap_oop(p);
! if (! oopDesc::is_null(o)) {
! oop obj1 = oopDesc::decode_heap_oop_not_null(o);
! if (_heap->in_collection_set(obj1)) {
! _has_cset_oops = true;
! }
! }
! }
public:
void do_oop(oop* o) {
! do_oop_work(o);
}
void do_oop(narrowOop* o) {
! do_oop_work(o);
}
! bool has_in_cset_oops() {
! return _has_cset_oops;
}
};
class ShenandoahNMethodOopInitializer : public OopClosure {
public:
--- 27,107 ----
#include "code/nmethod.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
! ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator() :
! _claimed_idx(0), _finished(false) {
! };
! void ShenandoahParallelCodeCacheIterator::parallel_blobs_do(CodeBlobClosure* f) {
! assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
!
! /*
! * Parallel code heap walk.
! *
! * This code makes all threads scan all code heaps, but only one thread would execute the
! * closure on given blob. This is achieved by recording the "claimed" blocks: if a thread
! * had claimed the block, it can process all blobs in it. Others have to fast-forward to
! * next attempt without processing.
! *
! * Late threads would return immediately if iterator is finished.
! */
!
! if (_finished) {
! return;
}
! int stride = 256; // educated guess
! int stride_mask = stride - 1;
! assert (is_power_of_2(stride), "sanity");
!
! int count = 0;
! bool process_block = true;
!
! for (CodeBlob *cb = CodeCache::first(); cb != NULL; cb = CodeCache::next(cb)) {
! int current = count++;
! if ((current & stride_mask) == 0) {
! process_block = (current >= _claimed_idx) &&
! (Atomic::cmpxchg(current + stride, &_claimed_idx, current) == current);
! }
! if (process_block) {
! if (cb->is_alive()) {
! f->do_code_blob(cb);
! #ifdef ASSERT
! if (cb->is_nmethod())
! ((nmethod*)cb)->verify_scavenge_root_oops();
! #endif
}
}
}
! _finished = true;
! }
+ class ShenandoahNMethodOopDetector : public OopClosure {
private:
! ShenandoahHeap* _heap;
! GrowableArray<oop*> _oops;
public:
+ ShenandoahNMethodOopDetector() : _heap(ShenandoahHeap::heap()), _oops(10) {};
+
void do_oop(oop* o) {
! _oops.append(o);
}
+
void do_oop(narrowOop* o) {
! fatal("NMethods should not have compressed oops embedded.");
}
!
! GrowableArray<oop*>* oops() {
! return &_oops;
! }
!
! bool has_oops() {
! return !_oops.is_empty();
}
};
class ShenandoahNMethodOopInitializer : public OopClosure {
public:
*** 96,106 ****
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj1 = oopDesc::decode_heap_oop_not_null(o);
oop obj2 = oopDesc::bs()->write_barrier(obj1);
if (! oopDesc::unsafe_equals(obj1, obj2)) {
! assert (!ShenandoahHeap::heap()->in_collection_set(obj2), "sanity");
oopDesc::encode_store_heap_oop(p, obj2);
}
}
}
--- 113,123 ----
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj1 = oopDesc::decode_heap_oop_not_null(o);
oop obj2 = oopDesc::bs()->write_barrier(obj1);
if (! oopDesc::unsafe_equals(obj1, obj2)) {
! shenandoah_assert_not_in_cset(NULL, obj2);
oopDesc::encode_store_heap_oop(p, obj2);
}
}
}
*** 111,126 ****
void do_oop(narrowOop* o) {
do_oop_work(o);
}
};
! volatile jint ShenandoahCodeRoots::_recorded_nmethods_lock;
! GrowableArray<nmethod*>* ShenandoahCodeRoots::_recorded_nmethods;
void ShenandoahCodeRoots::initialize() {
! _recorded_nmethods_lock = 0;
! _recorded_nmethods = new (ResourceObj::C_HEAP, mtGC) GrowableArray<nmethod*>(100, true, mtGC);
}
void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
--- 128,143 ----
void do_oop(narrowOop* o) {
do_oop_work(o);
}
};
! volatile jint ShenandoahCodeRoots::_recorded_nms_lock;
! GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;
void ShenandoahCodeRoots::initialize() {
! _recorded_nms_lock = 0;
! _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);
}
void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
*** 129,201 ****
nm->oops_do(&init);
nm->fix_oop_relocations();
break;
}
case 2: {
! fast_add_nmethod(nm);
break;
}
default:
ShouldNotReachHere();
}
};
void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
! case 1:
! break;
! case 2: {
! fast_remove_nmethod(nm);
break;
}
! default:
! ShouldNotReachHere();
! }
! }
!
! void ShenandoahCodeRoots::fast_add_nmethod(nmethod *nm) {
! ShenandoahNMethodCountOops count;
! nm->oops_do(&count);
! if (count.has_oops()) {
! ShenandoahNMethodOopInitializer init;
! nm->oops_do(&init);
! nm->fix_oop_relocations();
!
! ShenandoahCodeRootsLock lock(true);
! if (_recorded_nmethods->find(nm) == -1) {
! // Record methods once.
! _recorded_nmethods->append(nm);
! }
! }
! }
! void ShenandoahCodeRoots::fast_remove_nmethod(nmethod* nm) {
! ShenandoahNMethodCountOops count;
! nm->oops_do(&count, /* allow_zombie = */ true);
! if (count.has_oops()) {
ShenandoahCodeRootsLock lock(true);
! // GrowableArray::delete_at is O(1), which is exactly what we want.
! // TODO: Consider making _recorded_nmethods a HashTable to make find amortized O(1) too.
! int idx = _recorded_nmethods->find(nm);
assert(idx != -1, err_msg("nmethod " PTR_FORMAT " should be registered", p2i(nm)));
! _recorded_nmethods->delete_at(idx);
}
- };
-
- ShenandoahAllCodeRootsIterator ShenandoahCodeRoots::iterator() {
- return ShenandoahAllCodeRootsIterator();
- }
-
- ShenandoahCsetCodeRootsIterator ShenandoahCodeRoots::cset_iterator() {
- return ShenandoahCsetCodeRootsIterator();
}
ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
! _claimed(0), _heap(ShenandoahHeap::heap()),
! _par_iterator(CodeCache::parallel_iterator()) {
switch (ShenandoahCodeRootsStyle) {
case 0:
case 1:
break;
case 2:
--- 146,215 ----
nm->oops_do(&init);
nm->fix_oop_relocations();
break;
}
case 2: {
! ShenandoahNMethodOopDetector detector;
! nm->oops_do(&detector);
!
! if (detector.has_oops()) {
! ShenandoahNMethodOopInitializer init;
! nm->oops_do(&init);
! nm->fix_oop_relocations();
!
! ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
! nmr->assert_alive_and_correct();
!
! ShenandoahCodeRootsLock lock(true);
!
! int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
! if (idx != -1) {
! ShenandoahNMethod* old = _recorded_nms->at(idx);
! _recorded_nms->at_put(idx, nmr);
! delete old;
! } else {
! _recorded_nms->append(nmr);
! }
! }
break;
}
default:
ShouldNotReachHere();
}
};
void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
! case 1: {
break;
}
! case 2: {
! ShenandoahNMethodOopDetector detector;
! nm->oops_do(&detector, /* allow_zombie = */ true);
! if (detector.has_oops()) {
ShenandoahCodeRootsLock lock(true);
! int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
assert(idx != -1, err_msg("nmethod " PTR_FORMAT " should be registered", p2i(nm)));
! ShenandoahNMethod* old = _recorded_nms->at(idx);
! old->assert_same_oops(detector.oops());
! _recorded_nms->delete_at(idx);
! delete old;
! }
! break;
! }
! default:
! ShouldNotReachHere();
}
}
ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
! _heap(ShenandoahHeap::heap()),
! _claimed(0) {
! assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
switch (ShenandoahCodeRootsStyle) {
case 0:
case 1:
break;
case 2:
*** 207,280 ****
}
ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
switch (ShenandoahCodeRootsStyle) {
case 0:
! case 1:
break;
! case 2:
ShenandoahCodeRoots::release_lock(false);
break;
default:
ShouldNotReachHere();
}
}
template<bool CSET_FILTER>
void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
switch (ShenandoahCodeRootsStyle) {
! case 0:
if (_seq_claimed.try_set()) {
CodeCache::blobs_do(f);
}
break;
! case 1:
_par_iterator.parallel_blobs_do(f);
break;
case 2: {
! ShenandoahCodeRootsIterator::fast_parallel_blobs_do<false>(f);
break;
}
default:
ShouldNotReachHere();
}
}
void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
}
void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
}
template <bool CSET_FILTER>
void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
! assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at safepoint");
size_t stride = 256; // educated guess
! GrowableArray<nmethod *>* list = ShenandoahCodeRoots::_recorded_nmethods;
jlong max = list->length();
while (_claimed < max) {
! size_t cur = (size_t)(Atomic::add(stride, &_claimed) - stride);
size_t start = cur;
size_t end = MIN2(cur + stride, (size_t) max);
if (start >= (size_t)max) break;
for (size_t idx = start; idx < end; idx++) {
! nmethod *nm = list->at((int) idx);
! assert (nm->is_alive(), "only alive nmethods here");
! if (CSET_FILTER) {
! ShenandoahNMethodHasCSetOops scan(_heap);
! nm->oops_do(&scan);
! if (!scan.has_in_cset_oops()) {
continue;
}
}
! f->do_code_blob(nm);
}
}
}
--- 221,348 ----
}
ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
switch (ShenandoahCodeRootsStyle) {
case 0:
! case 1: {
! // No need to do anything here
break;
! }
! case 2: {
ShenandoahCodeRoots::release_lock(false);
break;
+ }
default:
ShouldNotReachHere();
}
}
template<bool CSET_FILTER>
void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
switch (ShenandoahCodeRootsStyle) {
! case 0: {
if (_seq_claimed.try_set()) {
CodeCache::blobs_do(f);
}
break;
! }
! case 1: {
_par_iterator.parallel_blobs_do(f);
break;
+ }
case 2: {
! ShenandoahCodeRootsIterator::fast_parallel_blobs_do<CSET_FILTER>(f);
break;
}
default:
ShouldNotReachHere();
}
}
+ ShenandoahAllCodeRootsIterator ShenandoahCodeRoots::iterator() {
+ return ShenandoahAllCodeRootsIterator();
+ }
+
+ ShenandoahCsetCodeRootsIterator ShenandoahCodeRoots::cset_iterator() {
+ return ShenandoahCsetCodeRootsIterator();
+ }
+
void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
}
void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
}
template <bool CSET_FILTER>
void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
! assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
size_t stride = 256; // educated guess
! GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;
jlong max = list->length();
while (_claimed < max) {
! size_t cur = (size_t)Atomic::add(stride, &_claimed); // Note: in JDK 8, add() returns old value
size_t start = cur;
size_t end = MIN2(cur + stride, (size_t) max);
if (start >= (size_t)max) break;
for (size_t idx = start; idx < end; idx++) {
! ShenandoahNMethod* nmr = list->at((int) idx);
! nmr->assert_alive_and_correct();
! if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {
continue;
}
+
+ f->do_code_blob(nmr->nm());
+ }
+ }
+ }
+
+ ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {
+ _nm = nm;
+ _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);
+ _oops_count = oops->length();
+ for (int c = 0; c < _oops_count; c++) {
+ _oops[c] = oops->at(c);
+ }
+ }
+
+ ShenandoahNMethod::~ShenandoahNMethod() {
+ if (_oops != NULL) {
+ FREE_C_HEAP_ARRAY(oop*, _oops, mtGC);
+ }
+ }
+
+ bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
+ for (int c = 0; c < _oops_count; c++) {
+ oop o = oopDesc::load_heap_oop(_oops[c]);
+ if (heap->in_collection_set(o)) {
+ return true;
+ }
}
+ return false;
+ }
! #ifdef ASSERT
! void ShenandoahNMethod::assert_alive_and_correct() {
! assert(_nm->is_alive(), "only alive nmethods here");
! assert(_oops_count > 0, "should have filtered nmethods without oops before");
! ShenandoahHeap* heap = ShenandoahHeap::heap();
! for (int c = 0; c < _oops_count; c++) {
! oop o = oopDesc::load_heap_oop(_oops[c]);
! shenandoah_assert_correct_except(NULL, o, o == NULL || heap->is_full_gc_move_in_progress());
! assert(_nm->code_contains((address)_oops[c]) || _nm->oops_contains(_oops[c]), "nmethod should contain the oop*");
}
+ }
+
+ void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {
+ assert(_oops_count == oops->length(), "should have the same number of oop*");
+ for (int c = 0; c < _oops_count; c++) {
+ assert(_oops[c] == oops->at(c), "should be the same oop*");
}
}
+ #endif
< prev index next >