< prev index next >
src/share/vm/gc_implementation/shenandoah/shenandoahCodeRoots.cpp
Print this page
rev 10509 : [backport] Refactor and improve ShenandoahCodeRoots strategies
rev 10598 : [backport] Shenandoah changes to allow enabling -Wreorder
rev 10600 : [backport] Trivial enhancement to avoid costly deletion array element
rev 10614 : [backport] Replace custom asserts with shenandoah_assert_*
rev 10619 : [backport] Move ParallelCodeIterator to ShenandoahCodeRoots
@@ -27,64 +27,81 @@
#include "code/nmethod.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
-class ShenandoahNMethodCountOops : public OopClosure {
-public:
- size_t _non_null_oops;
- ShenandoahNMethodCountOops() : _non_null_oops(0) {};
+ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator() :
+ _claimed_idx(0), _finished(false) {
+};
-private:
- template <class T>
- inline void do_oop_work(T* p) {
- T o = oopDesc::load_heap_oop(p);
- if (! oopDesc::is_null(o)) {
- _non_null_oops++;
- }
+void ShenandoahParallelCodeCacheIterator::parallel_blobs_do(CodeBlobClosure* f) {
+ assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
+
+ /*
+ * Parallel code heap walk.
+ *
+ * This code makes all threads scan all code heaps, but only one thread would execute the
+ * closure on given blob. This is achieved by recording the "claimed" blocks: if a thread
+ * had claimed the block, it can process all blobs in it. Others have to fast-forward to
+ * next attempt without processing.
+ *
+ * Late threads would return immediately if iterator is finished.
+ */
+
+ if (_finished) {
+ return;
}
-public:
- void do_oop(oop* o) {
- do_oop_work(o);
+ int stride = 256; // educated guess
+ int stride_mask = stride - 1;
+ assert (is_power_of_2(stride), "sanity");
+
+ int count = 0;
+ bool process_block = true;
+
+ for (CodeBlob *cb = CodeCache::first(); cb != NULL; cb = CodeCache::next(cb)) {
+ int current = count++;
+ if ((current & stride_mask) == 0) {
+ process_block = (current >= _claimed_idx) &&
+ (Atomic::cmpxchg(current + stride, &_claimed_idx, current) == current);
+ }
+ if (process_block) {
+ if (cb->is_alive()) {
+ f->do_code_blob(cb);
+#ifdef ASSERT
+ if (cb->is_nmethod())
+ ((nmethod*)cb)->verify_scavenge_root_oops();
+#endif
}
- void do_oop(narrowOop* o) {
- do_oop_work(o);
}
- bool has_oops() {
- return _non_null_oops > 0;
}
-};
-class ShenandoahNMethodHasCSetOops : public OopClosure {
-public:
- ShenandoahHeap* _heap;
- bool _has_cset_oops;
- ShenandoahNMethodHasCSetOops(ShenandoahHeap* heap) : _heap(heap), _has_cset_oops(false) {};
+ _finished = true;
+}
+class ShenandoahNMethodOopDetector : public OopClosure {
private:
- template <class T>
- inline void do_oop_work(T* p) {
- if (_has_cset_oops) return;
- T o = oopDesc::load_heap_oop(p);
- if (! oopDesc::is_null(o)) {
- oop obj1 = oopDesc::decode_heap_oop_not_null(o);
- if (_heap->in_collection_set(obj1)) {
- _has_cset_oops = true;
- }
- }
- }
+ ShenandoahHeap* _heap;
+ GrowableArray<oop*> _oops;
public:
+ ShenandoahNMethodOopDetector() : _heap(ShenandoahHeap::heap()), _oops(10) {};
+
void do_oop(oop* o) {
- do_oop_work(o);
+ _oops.append(o);
}
+
void do_oop(narrowOop* o) {
- do_oop_work(o);
+ fatal("NMethods should not have compressed oops embedded.");
}
- bool has_in_cset_oops() {
- return _has_cset_oops;
+
+ GrowableArray<oop*>* oops() {
+ return &_oops;
+ }
+
+ bool has_oops() {
+ return !_oops.is_empty();
}
};
class ShenandoahNMethodOopInitializer : public OopClosure {
public:
@@ -96,11 +113,11 @@
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj1 = oopDesc::decode_heap_oop_not_null(o);
oop obj2 = oopDesc::bs()->write_barrier(obj1);
if (! oopDesc::unsafe_equals(obj1, obj2)) {
- assert (!ShenandoahHeap::heap()->in_collection_set(obj2), "sanity");
+ shenandoah_assert_not_in_cset(NULL, obj2);
oopDesc::encode_store_heap_oop(p, obj2);
}
}
}
@@ -111,16 +128,16 @@
void do_oop(narrowOop* o) {
do_oop_work(o);
}
};
-volatile jint ShenandoahCodeRoots::_recorded_nmethods_lock;
-GrowableArray<nmethod*>* ShenandoahCodeRoots::_recorded_nmethods;
+volatile jint ShenandoahCodeRoots::_recorded_nms_lock;
+GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;
void ShenandoahCodeRoots::initialize() {
- _recorded_nmethods_lock = 0;
- _recorded_nmethods = new (ResourceObj::C_HEAP, mtGC) GrowableArray<nmethod*>(100, true, mtGC);
+ _recorded_nms_lock = 0;
+ _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);
}
void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
@@ -129,73 +146,70 @@
nm->oops_do(&init);
nm->fix_oop_relocations();
break;
}
case 2: {
- fast_add_nmethod(nm);
+ ShenandoahNMethodOopDetector detector;
+ nm->oops_do(&detector);
+
+ if (detector.has_oops()) {
+ ShenandoahNMethodOopInitializer init;
+ nm->oops_do(&init);
+ nm->fix_oop_relocations();
+
+ ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
+ nmr->assert_alive_and_correct();
+
+ ShenandoahCodeRootsLock lock(true);
+
+ int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
+ if (idx != -1) {
+ ShenandoahNMethod* old = _recorded_nms->at(idx);
+ _recorded_nms->at_put(idx, nmr);
+ delete old;
+ } else {
+ _recorded_nms->append(nmr);
+ }
+ }
break;
}
default:
ShouldNotReachHere();
}
};
void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
- case 1:
- break;
- case 2: {
- fast_remove_nmethod(nm);
+ case 1: {
break;
}
- default:
- ShouldNotReachHere();
- }
-}
-
-void ShenandoahCodeRoots::fast_add_nmethod(nmethod *nm) {
- ShenandoahNMethodCountOops count;
- nm->oops_do(&count);
- if (count.has_oops()) {
- ShenandoahNMethodOopInitializer init;
- nm->oops_do(&init);
- nm->fix_oop_relocations();
-
- ShenandoahCodeRootsLock lock(true);
- if (_recorded_nmethods->find(nm) == -1) {
- // Record methods once.
- _recorded_nmethods->append(nm);
- }
- }
-}
+ case 2: {
+ ShenandoahNMethodOopDetector detector;
+ nm->oops_do(&detector, /* allow_zombie = */ true);
-void ShenandoahCodeRoots::fast_remove_nmethod(nmethod* nm) {
- ShenandoahNMethodCountOops count;
- nm->oops_do(&count, /* allow_zombie = */ true);
- if (count.has_oops()) {
+ if (detector.has_oops()) {
ShenandoahCodeRootsLock lock(true);
- // GrowableArray::delete_at is O(1), which is exactly what we want.
- // TODO: Consider making _recorded_nmethods a HashTable to make find amortized O(1) too.
- int idx = _recorded_nmethods->find(nm);
+ int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
assert(idx != -1, err_msg("nmethod " PTR_FORMAT " should be registered", p2i(nm)));
- _recorded_nmethods->delete_at(idx);
+ ShenandoahNMethod* old = _recorded_nms->at(idx);
+ old->assert_same_oops(detector.oops());
+ _recorded_nms->delete_at(idx);
+ delete old;
+ }
+ break;
+ }
+ default:
+ ShouldNotReachHere();
}
-};
-
-ShenandoahAllCodeRootsIterator ShenandoahCodeRoots::iterator() {
- return ShenandoahAllCodeRootsIterator();
-}
-
-ShenandoahCsetCodeRootsIterator ShenandoahCodeRoots::cset_iterator() {
- return ShenandoahCsetCodeRootsIterator();
}
ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
- _claimed(0), _heap(ShenandoahHeap::heap()),
- _par_iterator(CodeCache::parallel_iterator()) {
+ _heap(ShenandoahHeap::heap()),
+ _claimed(0) {
+ assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
switch (ShenandoahCodeRootsStyle) {
case 0:
case 1:
break;
case 2:
@@ -207,74 +221,128 @@
}
ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
switch (ShenandoahCodeRootsStyle) {
case 0:
- case 1:
+ case 1: {
+ // No need to do anything here
break;
- case 2:
+ }
+ case 2: {
ShenandoahCodeRoots::release_lock(false);
break;
+ }
default:
ShouldNotReachHere();
}
}
template<bool CSET_FILTER>
void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
switch (ShenandoahCodeRootsStyle) {
- case 0:
+ case 0: {
if (_seq_claimed.try_set()) {
CodeCache::blobs_do(f);
}
break;
- case 1:
+ }
+ case 1: {
_par_iterator.parallel_blobs_do(f);
break;
+ }
case 2: {
- ShenandoahCodeRootsIterator::fast_parallel_blobs_do<false>(f);
+ ShenandoahCodeRootsIterator::fast_parallel_blobs_do<CSET_FILTER>(f);
break;
}
default:
ShouldNotReachHere();
}
}
+ShenandoahAllCodeRootsIterator ShenandoahCodeRoots::iterator() {
+ return ShenandoahAllCodeRootsIterator();
+}
+
+ShenandoahCsetCodeRootsIterator ShenandoahCodeRoots::cset_iterator() {
+ return ShenandoahCsetCodeRootsIterator();
+}
+
void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
}
void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
}
template <bool CSET_FILTER>
void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
- assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at safepoint");
+ assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
size_t stride = 256; // educated guess
- GrowableArray<nmethod *>* list = ShenandoahCodeRoots::_recorded_nmethods;
+ GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;
jlong max = list->length();
while (_claimed < max) {
- size_t cur = (size_t)(Atomic::add(stride, &_claimed) - stride);
+ size_t cur = (size_t)Atomic::add(stride, &_claimed); // Note: in JDK 8, add() returns old value
size_t start = cur;
size_t end = MIN2(cur + stride, (size_t) max);
if (start >= (size_t)max) break;
for (size_t idx = start; idx < end; idx++) {
- nmethod *nm = list->at((int) idx);
- assert (nm->is_alive(), "only alive nmethods here");
+ ShenandoahNMethod* nmr = list->at((int) idx);
+ nmr->assert_alive_and_correct();
- if (CSET_FILTER) {
- ShenandoahNMethodHasCSetOops scan(_heap);
- nm->oops_do(&scan);
- if (!scan.has_in_cset_oops()) {
+ if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {
continue;
}
+
+ f->do_code_blob(nmr->nm());
+ }
+ }
+}
+
+ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {
+ _nm = nm;
+ _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);
+ _oops_count = oops->length();
+ for (int c = 0; c < _oops_count; c++) {
+ _oops[c] = oops->at(c);
+ }
+}
+
+ShenandoahNMethod::~ShenandoahNMethod() {
+ if (_oops != NULL) {
+ FREE_C_HEAP_ARRAY(oop*, _oops, mtGC);
+ }
+}
+
+bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
+ for (int c = 0; c < _oops_count; c++) {
+ oop o = oopDesc::load_heap_oop(_oops[c]);
+ if (heap->in_collection_set(o)) {
+ return true;
+ }
}
+ return false;
+}
- f->do_code_blob(nm);
+#ifdef ASSERT
+void ShenandoahNMethod::assert_alive_and_correct() {
+ assert(_nm->is_alive(), "only alive nmethods here");
+ assert(_oops_count > 0, "should have filtered nmethods without oops before");
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ for (int c = 0; c < _oops_count; c++) {
+ oop o = oopDesc::load_heap_oop(_oops[c]);
+ shenandoah_assert_correct_except(NULL, o, o == NULL || heap->is_full_gc_move_in_progress());
+ assert(_nm->code_contains((address)_oops[c]) || _nm->oops_contains(_oops[c]), "nmethod should contain the oop*");
}
+}
+
+void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {
+ assert(_oops_count == oops->length(), "should have the same number of oop*");
+ for (int c = 0; c < _oops_count; c++) {
+ assert(_oops[c] == oops->at(c), "should be the same oop*");
}
}
+#endif
< prev index next >