--- old/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp 2019-11-11 16:09:50.022343268 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp 2019-11-11 16:09:49.936343273 +0100 @@ -46,6 +46,7 @@ #include "shenandoahBarrierSetAssembler_stub.hpp" #endif +template class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure { private: ShenandoahHeap* _heap; @@ -53,14 +54,23 @@ template inline void do_oop_work(T* p) { - _heap->maybe_update_with_forwarded(p); + oop o; + if (STOREVAL_EVAC_BARRIER) { + o = _heap->evac_update_with_forwarded(p); + if (!oopDesc::is_null(o)) { + _bs->enqueue(o); + } + } else { + _heap->maybe_update_with_forwarded(p); + } } public: ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()), _bs(ShenandoahBarrierSet::barrier_set()) { assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); } - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } + + virtual void do_oop(oop* p) { do_oop_work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } }; ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : @@ -150,10 +160,10 @@ // return *v; } -template +template void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) { - assert(UseShenandoahGC && ShenandoahCloneBarrier, "Should be enabled"); - ShenandoahUpdateRefsForOopClosure cl; + assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); + ShenandoahUpdateRefsForOopClosure cl; T* dst = (T*) start; for (size_t i = 0; i < count; i++) { cl.do_oop(dst++); @@ -165,11 +175,19 @@ if (!ShenandoahCloneBarrier) return; if (!need_update_refs_barrier()) return; - ShenandoahEvacOOMScope oom_evac_scope; - if (UseCompressedOops) { - write_ref_array_loop(start, count); + if (_heap->is_concurrent_traversal_in_progress()) { + ShenandoahEvacOOMScope oom_evac_scope; + if (UseCompressedOops) { + write_ref_array_loop(start, count); + } else { + write_ref_array_loop(start, count); + } } else { - write_ref_array_loop(start, count); + if (UseCompressedOops) { + write_ref_array_loop(start, count); + } else { + write_ref_array_loop(start, count); + } } } @@ -202,28 +220,25 @@ } template -void ShenandoahBarrierSet::write_ref_field_pre_static(T* field, oop newVal) { - T heap_oop = oopDesc::load_heap_oop(field); - - shenandoah_assert_not_in_cset_loc_except(field, ShenandoahHeap::heap()->cancelled_gc()); - - if (!oopDesc::is_null(heap_oop)) { - ShenandoahBarrierSet::barrier_set()->enqueue(oopDesc::decode_heap_oop(heap_oop)); - } -} - -template inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) { - write_ref_field_pre_static(field, newVal); + newVal = load_reference_barrier(newVal); + storeval_barrier(newVal); + if (ShenandoahSATBBarrier) { + T heap_oop = oopDesc::load_heap_oop(field); + shenandoah_assert_not_in_cset_loc_except(field, ShenandoahHeap::heap()->cancelled_gc()); + if (!oopDesc::is_null(heap_oop)) { + ShenandoahBarrierSet::barrier_set()->enqueue(oopDesc::decode_heap_oop(heap_oop)); + } + } } // These are the more general virtual versions. void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) { - write_ref_field_pre_static(field, new_val); + inline_write_ref_field_pre(field, new_val); } void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) { - write_ref_field_pre_static(field, new_val); + inline_write_ref_field_pre(field, new_val); } void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) { @@ -242,11 +257,16 @@ // it would be NULL in any case. But we *are* interested in any oop* // that potentially need to be updated. - ShenandoahEvacOOMScope oom_evac_scope; oop obj = oop(mr.start()); shenandoah_assert_correct(NULL, obj); - ShenandoahUpdateRefsForOopClosure cl; - obj->oop_iterate(&cl); + if (_heap->is_concurrent_traversal_in_progress()) { + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } else { + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } } oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj) { @@ -269,7 +289,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) { assert(ShenandoahLoadRefBarrier, "should be enabled"); - assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION), "evac should be in progress"); + assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL), "evac should be in progress"); shenandoah_assert_in_cset(NULL, obj); oop fwd = resolve_forwarded_not_null(obj); @@ -289,7 +309,10 @@ size_t max = ShenandoahEvacAssist; if (max > 0) { - ShenandoahMarkingContext* ctx = _heap->complete_marking_context(); + // Traversal is special: it uses incomplete marking context, because it coalesces evac with mark. + // Other code uses complete marking context, because evac happens after the mark. + ShenandoahMarkingContext* ctx = _heap->is_concurrent_traversal_in_progress() ? + _heap->marking_context() : _heap->complete_marking_context(); ShenandoahHeapRegion* r = _heap->heap_region_containing(obj); assert(r->is_cset(), "sanity"); @@ -314,7 +337,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj) { assert(ShenandoahLoadRefBarrier, "should be enabled"); if (!oopDesc::is_null(obj)) { - bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION); + bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); oop fwd = resolve_forwarded_not_null(obj); if (evac_in_progress && _heap->in_collection_set(obj) && @@ -334,7 +357,21 @@ } } +void ShenandoahBarrierSet::storeval_barrier(oop obj) { + if (ShenandoahStoreValEnqueueBarrier && !oopDesc::is_null(obj) && _heap->is_concurrent_traversal_in_progress()) { + enqueue(obj); + } +} + +void ShenandoahBarrierSet::keep_alive_barrier(oop obj) { + if (ShenandoahKeepAliveBarrier && _heap->is_concurrent_mark_in_progress()) { + enqueue(obj); + } +} + void ShenandoahBarrierSet::enqueue(oop obj) { + shenandoah_assert_not_forwarded_if(NULL, obj, _heap->is_concurrent_traversal_in_progress()); + // Filter marked objects before hitting the SATB queues. The same predicate would // be used by SATBMQ::filter to eliminate already marked objects downstream, but // filtering here helps to avoid wasteful SATB queueing work to begin with. @@ -371,5 +408,8 @@ if (ShenandoahSATBBarrier && success && result != NULL) { enqueue(result); } + if (new_value != NULL) { + storeval_barrier(new_value); + } return result; }