# HG changeset patch # User rkennke # Date 1559213612 -7200 # Thu May 30 12:53:32 2019 +0200 # Node ID e15383164fe2879daa32f8a598137f7fc27bc75d # Parent 87e545ffbce667ece86d02fb02dfec4fcee5be02 Streamline CAS barriers, C2 ideal graph generation and matching diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -1897,6 +1897,10 @@ case Op_CompareAndSwapN: // case Op_CompareAndSwapB: // case Op_CompareAndSwapS: +#if INCLUDE_SHENANDOAHGC + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: +#endif return true; // These are TBD case Op_WeakCompareAndSwapB: @@ -9670,7 +9674,6 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR); match(Set res (CompareAndSwapP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); @@ -9689,8 +9692,7 @@ instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ - predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR); - match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(TEMP tmp, KILL cr); @@ -9706,7 +9708,6 @@ instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{ - predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR); match(Set res (CompareAndSwapN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); @@ -9725,8 +9726,7 @@ instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ - predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR); - match(Set res (CompareAndSwapN mem (Binary oldval newval))); + match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(TEMP tmp, KILL cr); @@ -9786,7 +9786,7 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR)); + predicate(needs_acquiring_load_exclusive(n)); match(Set res (CompareAndSwapP mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); @@ -9805,8 +9805,8 @@ instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR); - match(Set res (CompareAndSwapP mem (Binary oldval newval))); + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); effect(TEMP tmp, KILL cr); @@ -9822,7 +9822,7 @@ instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier|| n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR)); + predicate(needs_acquiring_load_exclusive(n)); match(Set res (CompareAndSwapN mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); @@ -9841,8 +9841,8 @@ instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR); - match(Set res (CompareAndSwapN mem (Binary oldval newval))); + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); effect(TEMP tmp, KILL cr); @@ -9939,7 +9939,6 @@ %} instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ - predicate(!UseShenandoahGC || !ShenandoahCASBarrier); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr); @@ -9955,8 +9954,7 @@ %} instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ - predicate(UseShenandoahGC && ShenandoahCASBarrier); - match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval))); ins_cost(3 * VOLATILE_REF_COST); effect(TEMP_DEF res, TEMP tmp, KILL cr); format %{ @@ -9972,7 +9970,6 @@ %} instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR); match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr); @@ -9988,8 +9985,7 @@ %} instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ - predicate(UseShenandoahGC && ShenandoahCASBarrier); - match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval))); ins_cost(3 * VOLATILE_REF_COST); effect(TEMP_DEF res, TEMP tmp, KILL cr); format %{ @@ -10073,7 +10069,6 @@ %} instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ - predicate(!UseShenandoahGC || !ShenandoahCASBarrier); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); @@ -10091,8 +10086,7 @@ %} instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ - predicate(UseShenandoahGC && ShenandoahCASBarrier); - match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval))); ins_cost(3 * VOLATILE_REF_COST); effect(TEMP tmp, KILL cr); format %{ @@ -10108,7 +10102,6 @@ %} instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR); match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); @@ -10126,8 +10119,7 @@ %} instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ - predicate(UseShenandoahGC && ShenandoahCASBarrier); - match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval))); ins_cost(3 * VOLATILE_REF_COST); effect(TEMP tmp, KILL cr); format %{ diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -7623,7 +7623,7 @@ rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ - predicate(VM_Version::supports_cx8() && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR)); + predicate(VM_Version::supports_cx8()); match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval))); match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval))); effect(KILL cr, KILL oldval); @@ -7649,19 +7649,20 @@ rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ - predicate(VM_Version::supports_cx8() && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR); - match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval))); - match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval))); + predicate(VM_Version::supports_cx8()); + match(Set res (ShenandoahCompareAndSwapP mem_ptr (Binary oldval newval))); + match(Set res (ShenandoahWeakCompareAndSwapP mem_ptr (Binary oldval newval))); effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval); format %{ "shenandoah_cas_oop $mem_ptr,$newval" %} ins_encode %{ #if INCLUDE_SHENANDOAHGC - ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, - false, // swap - $tmp1$$Register, $tmp2$$Register - ); + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, + $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, + false, // swap + $tmp1$$Register, $tmp2$$Register + ); #else ShouldNotReachHere(); #endif @@ -7771,7 +7772,6 @@ memory mem_ptr, rax_RegN oldval, rRegN newval, rFlagsReg cr) %{ - predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR); match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval))); match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval))); effect(KILL cr, KILL oldval); @@ -7796,19 +7796,19 @@ rRegP tmp1, rRegP tmp2, rax_RegN oldval, rRegN newval, rFlagsReg cr) %{ - predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR); - match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval))); - match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval))); + match(Set res (ShenandoahCompareAndSwapN mem_ptr (Binary oldval newval))); + match(Set res (ShenandoahWeakCompareAndSwapN mem_ptr (Binary oldval newval))); effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval); format %{ "shenandoah_cas_oop $mem_ptr,$newval" %} ins_encode %{ #if INCLUDE_SHENANDOAHGC - ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, - false, // swap - $tmp1$$Register, $tmp2$$Register - ); + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, + $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, + false, // swap + $tmp1$$Register, $tmp2$$Register + ); #else ShouldNotReachHere(); #endif @@ -7898,7 +7898,6 @@ memory mem_ptr, rax_RegN oldval, rRegN newval, rFlagsReg cr) %{ - predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR); match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval))); effect(KILL cr); @@ -7917,18 +7916,18 @@ rax_RegN oldval, rRegN newval, rRegP tmp1, rRegP tmp2, rFlagsReg cr) %{ - predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR); - match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval))); + match(Set oldval (ShenandoahCompareAndExchangeN mem_ptr (Binary oldval newval))); effect(TEMP tmp1, TEMP tmp2, KILL cr); format %{ "shenandoah_cas_oop $mem_ptr,$newval" %} ins_encode %{ #if INCLUDE_SHENANDOAHGC - ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, - true, // exchange - $tmp1$$Register, $tmp2$$Register - ); + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, + NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, + true, // exchange + $tmp1$$Register, $tmp2$$Register + ); #else ShouldNotReachHere(); #endif @@ -7941,7 +7940,7 @@ rax_RegP oldval, rRegP newval, rFlagsReg cr) %{ - predicate(VM_Version::supports_cx8() && (!UseShenandoahGC || ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR)); + predicate(VM_Version::supports_cx8()); match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval))); effect(KILL cr); @@ -7961,8 +7960,8 @@ rRegP tmp1, rRegP tmp2, rFlagsReg cr) %{ - predicate(VM_Version::supports_cx8() && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR); - match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval))); + predicate(VM_Version::supports_cx8()); + match(Set oldval (ShenandoahCompareAndExchangeP mem_ptr (Binary oldval newval))); effect(KILL cr, TEMP tmp1, TEMP tmp2); ins_cost(1000); @@ -7970,10 +7969,11 @@ ins_encode %{ #if INCLUDE_SHENANDOAHGC - ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, - true, // exchange - $tmp1$$Register, $tmp2$$Register - ); + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, + NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, + true, // exchange + $tmp1$$Register, $tmp2$$Register + ); #else ShouldNotReachHere(); #endif diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp --- a/src/hotspot/share/adlc/formssel.cpp +++ b/src/hotspot/share/adlc/formssel.cpp @@ -3516,6 +3516,9 @@ "CompareAndSwapB", "CompareAndSwapS", "CompareAndSwapI", "CompareAndSwapL", "CompareAndSwapP", "CompareAndSwapN", "WeakCompareAndSwapB", "WeakCompareAndSwapS", "WeakCompareAndSwapI", "WeakCompareAndSwapL", "WeakCompareAndSwapP", "WeakCompareAndSwapN", "CompareAndExchangeB", "CompareAndExchangeS", "CompareAndExchangeI", "CompareAndExchangeL", "CompareAndExchangeP", "CompareAndExchangeN", +#if INCLUDE_SHENANDOAHGC + "ShenandoahCompareAndSwapN", "ShenandoahCompareAndSwapP", "ShenandoahWeakCompareAndSwapP", "ShenandoahWeakCompareAndSwapN", "ShenandoahCompareAndExchangeP", "ShenandoahCompareAndExchangeN", +#endif "StoreCM", "ClearArray", "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP", diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -625,41 +625,120 @@ return load; } +static void pin_atomic_op(C2AtomicAccess& access) { + if (!access.needs_pinning()) { + return; + } + // SCMemProjNodes represent the memory state of a LoadStore. Their + // main role is to prevent LoadStore nodes from being optimized away + // when their results aren't used. + GraphKit* kit = access.kit(); + Node* load_store = access.raw_access(); + assert(load_store != NULL, "must pin atomic op"); + Node* proj = kit->gvn().transform(new SCMemProjNode(load_store)); + kit->set_memory(proj, access.alias_idx()); +} + Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, - Node* val, const Type* value_type) const { + Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); if (access.is_oop()) { - val = shenandoah_storeval_barrier(kit, val); + new_val = shenandoah_storeval_barrier(kit, new_val); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, expected_val /* pre_val */, T_OBJECT); - } - return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, val, value_type); -} + MemNode::MemOrd mo = access.mem_node_mo(); + Node* mem = access.memory(); + Node* adr = access.addr().node(); + const TypePtr* adr_type = access.addr().type(); + Node* load_store = NULL; -Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::atomic_cmpxchg_val_at(access, expected_val, new_val, val_type); +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); + if (ShenandoahCASBarrier) { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } else { + load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } + } else +#endif + { + if (ShenandoahCASBarrier) { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } else { + load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } + } + + access.set_raw_access(load_store); + pin_atomic_op(access); + +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + } +#endif + return load_store; + } + return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); } Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, - Node* val, const Type* value_type) const { + Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); if (access.is_oop()) { - val = shenandoah_storeval_barrier(kit, val); + new_val = shenandoah_storeval_barrier(kit, new_val); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, expected_val /* pre_val */, T_OBJECT); + DecoratorSet decorators = access.decorators(); + MemNode::MemOrd mo = access.mem_node_mo(); + Node* mem = access.memory(); + bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0; + Node* load_store = NULL; + Node* adr = access.addr().node(); +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); + if (ShenandoahCASBarrier) { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } + } else { + if (is_weak_cas) { + load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } + } + } else +#endif + { + if (ShenandoahCASBarrier) { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } + } else { + if (is_weak_cas) { + load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } + } + } + access.set_raw_access(load_store); + pin_atomic_op(access); + return load_store; } - return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, val, value_type); -} - -Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::atomic_cmpxchg_bool_at(access, expected_val, new_val, val_type); + return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); } Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* val, const Type* value_type) const { @@ -676,11 +755,6 @@ return result; } -Node* ShenandoahBarrierSetC2::atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::atomic_xchg_at(access, new_val, value_type); -} - Node* ShenandoahBarrierSetC2::atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { // TODO: Implement using proper barriers. return BarrierSetC2::atomic_add_at(access, new_val, value_type); diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -104,11 +104,6 @@ virtual Node* store_at(C2Access& access, C2AccessValue& val) const; virtual Node* load_at(C2Access& access, const Type* val_type) const; - virtual Node* atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const; - virtual Node* atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const; - virtual Node* atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const; virtual Node* atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const; virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -170,8 +170,10 @@ n->Opcode() == Op_LoadN || n->Opcode() == Op_GetAndSetP || n->Opcode() == Op_CompareAndExchangeP || + n->Opcode() == Op_ShenandoahCompareAndExchangeP || n->Opcode() == Op_GetAndSetN || - n->Opcode() == Op_CompareAndExchangeN) { + n->Opcode() == Op_CompareAndExchangeN || + n->Opcode() == Op_ShenandoahCompareAndExchangeN) { return true; } if (n->Opcode() == Op_DecodeN || diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp @@ -289,4 +289,95 @@ void remove(Node* n); }; -#endif // SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAH_SUPPORT_HPP +class ShenandoahCompareAndSwapPNode : public CompareAndSwapPNode { +public: + ShenandoahCompareAndSwapPNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) + : CompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypePtr::NULL_PTR) { + return new CompareAndSwapPNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahCompareAndSwapNNode : public CompareAndSwapNNode { +public: + ShenandoahCompareAndSwapNNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) + : CompareAndSwapNNode(c, mem, adr, val, ex, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypeNarrowOop::NULL_PTR) { + return new CompareAndSwapNNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahWeakCompareAndSwapPNode : public WeakCompareAndSwapPNode { +public: + ShenandoahWeakCompareAndSwapPNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) + : WeakCompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypePtr::NULL_PTR) { + return new WeakCompareAndSwapPNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahWeakCompareAndSwapNNode : public WeakCompareAndSwapNNode { +public: + ShenandoahWeakCompareAndSwapNNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) + : WeakCompareAndSwapNNode(c, mem, adr, val, ex, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypeNarrowOop::NULL_PTR) { + return new WeakCompareAndSwapNNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahCompareAndExchangePNode : public CompareAndExchangePNode { +public: + ShenandoahCompareAndExchangePNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) + : CompareAndExchangePNode(c, mem, adr, val, ex, at, t, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypePtr::NULL_PTR) { + return new CompareAndExchangePNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), adr_type(), bottom_type(), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahCompareAndExchangeNNode : public CompareAndExchangeNNode { +public: + ShenandoahCompareAndExchangeNNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) + : CompareAndExchangeNNode(c, mem, adr, val, ex, at, t, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypeNarrowOop::NULL_PTR) { + return new CompareAndExchangeNNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), adr_type(), bottom_type(), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + + +#endif // SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -269,6 +269,12 @@ #else #define shmacro(x) optionalmacro(x) #endif +shmacro(ShenandoahCompareAndExchangeP) +shmacro(ShenandoahCompareAndExchangeN) +shmacro(ShenandoahCompareAndSwapN) +shmacro(ShenandoahCompareAndSwapP) +shmacro(ShenandoahWeakCompareAndSwapN) +shmacro(ShenandoahWeakCompareAndSwapP) shmacro(ShenandoahEnqueueBarrier) shmacro(ShenandoahReadBarrier) shmacro(ShenandoahWriteBarrier) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -3395,6 +3395,23 @@ } break; #if INCLUDE_SHENANDOAHGC + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: + case Op_ShenandoahWeakCompareAndSwapN: + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: +#ifdef ASSERT + if( VerifyOptoOopOffsets ) { + MemNode* mem = n->as_Mem(); + // Check to see if address types have grounded out somehow. + const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr(); + ciInstanceKlass *k = tp->klass()->as_instance_klass(); + bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); + assert( !tp || oop_offset_is_sane, "" ); + } +#endif + break; case Op_ShenandoahReadBarrier: break; case Op_ShenandoahWriteBarrier: diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -515,6 +515,10 @@ } case Op_CompareAndExchangeP: case Op_CompareAndExchangeN: +#if INCLUDE_SHENANDOAHGC + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: +#endif case Op_GetAndSetP: case Op_GetAndSetN: { add_objload_to_connection_graph(n, delayed_worklist); @@ -524,6 +528,12 @@ case Op_StoreN: case Op_StoreNKlass: case Op_StorePConditional: +#if INCLUDE_SHENANDOAHGC + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: +#endif case Op_WeakCompareAndSwapP: case Op_WeakCompareAndSwapN: case Op_CompareAndSwapP: @@ -762,6 +772,14 @@ case Op_CompareAndSwapN: case Op_WeakCompareAndSwapP: case Op_WeakCompareAndSwapN: +#if INCLUDE_SHENANDOAHGC + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: +#endif case Op_GetAndSetP: case Op_GetAndSetN: { Node* adr = n->in(MemNode::Address); @@ -775,6 +793,9 @@ } #endif if (opcode == Op_GetAndSetP || opcode == Op_GetAndSetN || +#if INCLUDE_SHENANDOAHGC + opcode == Op_ShenandoahCompareAndExchangeN || opcode == Op_ShenandoahCompareAndExchangeP || +#endif opcode == Op_CompareAndExchangeN || opcode == Op_CompareAndExchangeP) { add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL); } @@ -2145,6 +2166,10 @@ // Check for unsafe oop field access if (n->has_out_with(Op_StoreP, Op_LoadP, Op_StoreN, Op_LoadN) || n->has_out_with(Op_GetAndSetP, Op_GetAndSetN, Op_CompareAndExchangeP, Op_CompareAndExchangeN) || +#if INCLUDE_SHENANDOAHGC + n->has_out_with(Op_ShenandoahCompareAndExchangeP) || n->has_out_with(Op_ShenandoahCompareAndExchangeN) || + n->has_out_with(Op_ShenandoahCompareAndSwapP, Op_ShenandoahCompareAndSwapN, Op_ShenandoahWeakCompareAndSwapP, Op_ShenandoahWeakCompareAndSwapN) || +#endif n->has_out_with(Op_CompareAndSwapP, Op_CompareAndSwapN, Op_WeakCompareAndSwapP, Op_WeakCompareAndSwapN)) { bt = T_OBJECT; (*unsafe) = true; diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp --- a/src/hotspot/share/opto/matcher.cpp +++ b/src/hotspot/share/opto/matcher.cpp @@ -2263,6 +2263,14 @@ case Op_StorePConditional: case Op_StoreIConditional: case Op_StoreLConditional: +#if INCLUDE_SHENANDOAHGC + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: +#endif case Op_CompareAndExchangeB: case Op_CompareAndExchangeS: case Op_CompareAndExchangeI: @@ -2506,6 +2514,14 @@ // that a monitor exit operation contains a serializing instruction. if (xop == Op_MemBarVolatile || +#if INCLUDE_SHENANDOAHGC + xop == Op_ShenandoahCompareAndExchangeP || + xop == Op_ShenandoahCompareAndExchangeN || + xop == Op_ShenandoahWeakCompareAndSwapP || + xop == Op_ShenandoahWeakCompareAndSwapN || + xop == Op_ShenandoahCompareAndSwapN || + xop == Op_ShenandoahCompareAndSwapP || +#endif xop == Op_CompareAndExchangeB || xop == Op_CompareAndExchangeS || xop == Op_CompareAndExchangeI ||