# HG changeset patch # Parent 1791eaa73e810a987ddd5fe110c1fd39325e3aa6 diff -r 1791eaa73e81 src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Tue Jun 04 12:50:15 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Tue Jun 04 22:56:01 2019 +0200 @@ -473,9 +473,11 @@ } const TypeFunc* ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type() { - const Type **fields = TypeTuple::fields(1); + const Type **fields = TypeTuple::fields(2); fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); + fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM; // original field address + + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); // create result type (range) fields = TypeTuple::fields(1); @@ -544,7 +546,7 @@ if (access.is_oop()) { if (ShenandoahLoadRefBarrier) { - load = new ShenandoahLoadReferenceBarrierNode(NULL, load); + load = new ShenandoahLoadReferenceBarrierNode(NULL, load, adr); if (access.is_parse_access()) { load = static_cast(access).kit()->gvn().transform(load); } else { @@ -629,7 +631,7 @@ load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); } #endif - load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store)); + load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, kit->null())); return load_store; } return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); @@ -697,7 +699,7 @@ } Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type); if (access.is_oop()) { - result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result)); + result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, kit->null())); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, result /* pre_val */, T_OBJECT); @@ -1197,7 +1199,7 @@ } bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const { - return n->Opcode() == Op_ShenandoahLoadReferenceBarrier; + return false; //n->Opcode() == Op_ShenandoahLoadReferenceBarrier; } bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { diff -r 1791eaa73e81 src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp Tue Jun 04 12:50:15 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp Tue Jun 04 22:56:01 2019 +0200 @@ -1082,7 +1082,7 @@ phase->register_control(ctrl, loop, in_cset_fast_test_iff); } -void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* field_addr, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) { IdealLoopTree*loop = phase->get_loop(ctrl); const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); @@ -1093,13 +1093,14 @@ mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); phase->register_new_node(mm, ctrl); - Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM); + Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_selfheal_JRT), "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM); call->init_req(TypeFunc::Control, ctrl); call->init_req(TypeFunc::I_O, phase->C->top()); call->init_req(TypeFunc::Memory, mm); call->init_req(TypeFunc::FramePtr, phase->C->top()); call->init_req(TypeFunc::ReturnAdr, phase->C->top()); call->init_req(TypeFunc::Parms, val); + call->init_req(TypeFunc::Parms+1, field_addr); phase->register_control(call, loop, ctrl); ctrl = new ProjNode(call, TypeFunc::Control); phase->register_control(ctrl, loop, call); @@ -1464,7 +1465,7 @@ assert(val->bottom_type()->make_oopptr(), "need oop"); assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); - enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT }; + enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT }; Node* region = new RegionNode(PATH_LIMIT); Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); @@ -1514,49 +1515,11 @@ IfNode* iff = unc_ctrl->in(0)->as_If(); phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); } - Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes())); - phase->register_new_node(addr, ctrl); - assert(new_val->bottom_type()->isa_oopptr(), "what else?"); - Node* markword = new LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered); - phase->register_new_node(markword, ctrl); - - // Test if object is forwarded. This is the case if lowest two bits are set. - Node* masked = new AndXNode(markword, phase->igvn().MakeConX(markOopDesc::lock_mask_in_place)); - phase->register_new_node(masked, ctrl); - Node* cmp = new CmpXNode(masked, phase->igvn().MakeConX(markOopDesc::marked_value)); - phase->register_new_node(cmp, ctrl); - - // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr - Node* bol = new BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded - phase->register_new_node(bol, ctrl); - - IfNode* iff = new IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN); - phase->register_control(iff, loop, ctrl); - Node* if_fwd = new IfTrueNode(iff); - phase->register_control(if_fwd, loop, iff); - Node* if_not_fwd = new IfFalseNode(iff); - phase->register_control(if_not_fwd, loop, iff); - - // Decode forward pointer: since we already have the lowest bits, we can just subtract them - // from the mark word without the need for large immediate mask. - Node* masked2 = new SubXNode(markword, masked); - phase->register_new_node(masked2, if_fwd); - Node* fwdraw = new CastX2PNode(masked2); - fwdraw->init_req(0, if_fwd); - phase->register_new_node(fwdraw, if_fwd); - Node* fwd = new CheckCastPPNode(NULL, fwdraw, val->bottom_type()); - phase->register_new_node(fwd, if_fwd); - - // Wire up not-equal-path in slots 3. - region->init_req(_fwded, if_fwd); - val_phi->init_req(_fwded, fwd); - raw_mem_phi->init_req(_fwded, raw_mem); // Call lrb-stub and wire up that path in slots 4 Node* result_mem = NULL; - ctrl = if_not_fwd; - fwd = new_val; - call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase); + Node* fwd = new_val; + call_lrb_stub(ctrl, fwd, lrb->in(ShenandoahLoadReferenceBarrierNode::FieldAddr), result_mem, raw_mem, phase); region->init_req(_evac_path, ctrl); val_phi->init_req(_evac_path, fwd); raw_mem_phi->init_req(_evac_path, result_mem); @@ -2999,8 +2962,8 @@ } } -ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj) -: Node(ctrl, obj) { +ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, Node* field_addr) +: Node(ctrl, obj, field_addr) { ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); } diff -r 1791eaa73e81 src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp Tue Jun 04 12:50:15 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp Tue Jun 04 22:56:01 2019 +0200 @@ -60,7 +60,7 @@ static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase); static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, PhaseIdealLoop* phase); - static void call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase); + static void call_lrb_stub(Node*& ctrl, Node*& val, Node* field_addr, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase); static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase); static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase); @@ -227,14 +227,15 @@ public: enum { Control, - ValueIn + ValueIn, + FieldAddr }; enum Strength { NONE, WEAK, STRONG, NA }; - ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val); + ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val, Node* field_addr); virtual int Opcode() const; virtual const Type* bottom_type() const; diff -r 1791eaa73e81 src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp Tue Jun 04 12:50:15 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp Tue Jun 04 22:56:01 2019 +0200 @@ -169,7 +169,8 @@ // assert(CompressedOops::is_null(result) || // oopDesc::equals_raw(result, ShenandoahBarrierSet::resolve_oop_static_not_null(result)), // "expect not forwarded"); - return NULL; + return result; + //return NULL; } } else { shenandoah_assert_not_forwarded(p, heap_oop); diff -r 1791eaa73e81 src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Tue Jun 04 12:50:15 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Tue Jun 04 22:56:01 2019 +0200 @@ -60,6 +60,24 @@ return (oopDesc*) result; JRT_END +JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_selfheal_JRT(oopDesc* src, void* field_addr)) + oop orig = oop(src); + oop result = ShenandoahBarrierSet::resolve_forwarded_not_null(orig); + if (orig == result) { + result = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(orig); + } + if (field_addr != NULL && orig != result) { + // TODO: We want two entry points instead + // tty->print_cr("LRB fixing field at: " PTR_FORMAT " to " PTR_FORMAT, p2i(field_addr), p2i(result)); + if (UseCompressedOops) { + ShenandoahHeap::heap()->atomic_compare_exchange_oop(result, (narrowOop *) field_addr, orig); + } else { + ShenandoahHeap::heap()->atomic_compare_exchange_oop(result, (oop *) field_addr, orig); + } + } + return (oopDesc*) result; +JRT_END + // Shenandoah clone barrier: makes sure that references point to to-space // in cloned objects. JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* obj)) diff -r 1791eaa73e81 src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp Tue Jun 04 12:50:15 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp Tue Jun 04 22:56:01 2019 +0200 @@ -38,6 +38,7 @@ static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread); static oopDesc* load_reference_barrier_JRT(oopDesc* src); + static oopDesc* load_reference_barrier_selfheal_JRT(oopDesc* src, void* field); static void shenandoah_clone_barrier(oopDesc* obj); };