--- old/src/hotspot/share/ci/ciInstanceKlass.cpp 2018-10-26 11:07:46.748682906 +0200 +++ new/src/hotspot/share/ci/ciInstanceKlass.cpp 2018-10-26 11:07:38.190660634 +0200 @@ -554,6 +554,12 @@ _has_injected_fields = has_injected_fields; } +bool ciInstanceKlass::has_object_fields() const { + GUARDED_VM_ENTRY( + return get_instanceKlass()->nonstatic_oop_map_size() > 0; + ); +} + // ------------------------------------------------------------------ // ciInstanceKlass::find_method // --- old/src/hotspot/share/ci/ciInstanceKlass.hpp 2018-10-26 11:07:55.469705602 +0200 +++ new/src/hotspot/share/ci/ciInstanceKlass.hpp 2018-10-26 11:07:46.849683169 +0200 @@ -202,6 +202,8 @@ return _has_injected_fields > 0 ? true : false; } + bool has_object_fields() const; + // nth nonstatic field (presented by ascending address) ciField* nonstatic_field_at(int i) { assert(_nonstatic_fields != NULL, ""); --- old/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp 2018-10-26 11:08:04.012727835 +0200 +++ new/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp 2018-10-26 11:07:55.567705857 +0200 @@ -594,8 +594,6 @@ Node* G1BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { DecoratorSet decorators = access.decorators(); - GraphKit* kit = access.kit(); - Node* adr = access.addr().node(); Node* obj = access.base(); @@ -606,7 +604,8 @@ bool is_unordered = (decorators & MO_UNORDERED) != 0; bool need_cpu_mem_bar = !is_unordered || mismatched || !in_heap; - Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : kit->top(); + Node* top = Compile::current()->top(); + Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top; Node* load = CardTableBarrierSetC2::load_at_resolved(access, val_type); // If we are reading the value of the referent field of a Reference @@ -616,12 +615,16 @@ // Also we need to add memory barrier to prevent commoning reads // from this field across safepoint since GC can change its value. bool need_read_barrier = in_heap && (on_weak || - (unknown && offset != kit->top() && obj != kit->top())); + (unknown && offset != top && obj != top)); if (!access.is_oop() || !need_read_barrier) { return load; } + assert(access.is_parse_access(), "entry not supported at optimization time"); + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); + if (on_weak) { // Use the pre-barrier to record the value in the referent field pre_barrier(kit, false /* do_load */, --- old/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp 2018-10-26 11:08:12.589750156 +0200 +++ new/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp 2018-10-26 11:08:04.112728095 +0200 @@ -35,10 +35,12 @@ // By default this is a no-op. void BarrierSetC2::resolve_address(C2Access& access) const { } -void* C2Access::barrier_set_state() const { +void* C2ParseAccess::barrier_set_state() const { return _kit->barrier_set_state(); } +PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); } + bool C2Access::needs_cpu_membar() const { bool mismatched = (_decorators & C2_MISMATCHED) != 0; bool is_unordered = (_decorators & MO_UNORDERED) != 0; @@ -70,7 +72,6 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { DecoratorSet decorators = access.decorators(); - GraphKit* kit = access.kit(); bool mismatched = (decorators & C2_MISMATCHED) != 0; bool unaligned = (decorators & C2_UNALIGNED) != 0; @@ -79,22 +80,49 @@ bool in_native = (decorators & IN_NATIVE) != 0; assert(!in_native, "not supported yet"); - if (access.type() == T_DOUBLE) { - Node* new_val = kit->dstore_rounding(val.node()); - val.set_node(new_val); - } - MemNode::MemOrd mo = access.mem_node_mo(); + + Node* store; + if (access.is_parse_access()) { + C2ParseAccess& parse_access = static_cast(access); + + GraphKit* kit = parse_access.kit(); + if (access.type() == T_DOUBLE) { + Node* new_val = kit->dstore_rounding(val.node()); + val.set_node(new_val); + } - Node* store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(), + store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(), access.addr().type(), mo, requires_atomic_access, unaligned, mismatched); - access.set_raw_access(store); + access.set_raw_access(store); + } else { + assert(!requires_atomic_access, "not yet supported"); + assert(access.is_opt_access(), "either parse or opt access"); + C2OptAccess& opt_access = static_cast(access); + Node* ctl = opt_access.ctl(); + MergeMemNode* mm = opt_access.mem(); + PhaseGVN& gvn = opt_access.gvn(); + const TypePtr* adr_type = access.addr().type(); + int alias = gvn.C->get_alias_index(adr_type); + Node* mem = mm->memory_at(alias); + + StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), access.type(), mo); + if (unaligned) { + st->set_unaligned_access(); + } + if (mismatched) { + st->set_mismatched_access(); + } + store = gvn.transform(st); + if (store == st) { + mm->set_memory_at(alias, st); + } + } return store; } Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { DecoratorSet decorators = access.decorators(); - GraphKit* kit = access.kit(); Node* adr = access.addr().node(); const TypePtr* adr_type = access.addr().type(); @@ -109,16 +137,31 @@ MemNode::MemOrd mo = access.mem_node_mo(); LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest; - Node* control = control_dependent ? kit->control() : NULL; Node* load; - if (in_native) { - load = kit->make_load(control, adr, val_type, access.type(), mo); + if (access.is_parse_access()) { + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); + Node* control = control_dependent ? kit->control() : NULL; + + if (in_native) { + load = kit->make_load(control, adr, val_type, access.type(), mo); + } else { + load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, + dep, requires_atomic_access, unaligned, mismatched); + } + access.set_raw_access(load); } else { - load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, - dep, requires_atomic_access, unaligned, mismatched); + assert(!requires_atomic_access, "not yet supported"); + assert(access.is_opt_access(), "either parse or opt access"); + C2OptAccess& opt_access = static_cast(access); + Node* control = control_dependent ? opt_access.ctl() : NULL; + MergeMemNode* mm = opt_access.mem(); + PhaseGVN& gvn = opt_access.gvn(); + Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type)); + load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched); + load = gvn.transform(load); } - access.set_raw_access(load); return load; } @@ -130,7 +173,11 @@ public: C2AccessFence(C2Access& access) : _access(access), _leading_membar(NULL) { - GraphKit* kit = access.kit(); + GraphKit* kit = NULL; + if (access.is_parse_access()) { + C2ParseAccess& parse_access = static_cast(access); + kit = parse_access.kit(); + } DecoratorSet decorators = access.decorators(); bool is_write = (decorators & C2_WRITE_ACCESS) != 0; @@ -141,6 +188,7 @@ bool is_release = (decorators & MO_RELEASE) != 0; if (is_atomic) { + assert(kit != NULL, "unsupported at optimization time"); // Memory-model-wise, a LoadStore acts like a little synchronized // block, so needs barriers on each side. These don't translate // into actual barriers on most machines, but we still need rest of @@ -159,6 +207,7 @@ // floating down past the volatile write. Also prevents commoning // another volatile read. if (is_volatile || is_release) { + assert(kit != NULL, "unsupported at optimization time"); _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); } } else { @@ -168,11 +217,13 @@ // so there's no problems making a strong assert about mixing users // of safe & unsafe memory. if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) { + assert(kit != NULL, "unsupported at optimization time"); _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); } } if (access.needs_cpu_membar()) { + assert(kit != NULL, "unsupported at optimization time"); kit->insert_mem_bar(Op_MemBarCPUOrder); } @@ -185,7 +236,11 @@ } ~C2AccessFence() { - GraphKit* kit = _access.kit(); + GraphKit* kit = NULL; + if (_access.is_parse_access()) { + C2ParseAccess& parse_access = static_cast(_access); + kit = parse_access.kit(); + } DecoratorSet decorators = _access.decorators(); bool is_write = (decorators & C2_WRITE_ACCESS) != 0; @@ -202,6 +257,7 @@ } if (is_atomic) { + assert(kit != NULL, "unsupported at optimization time"); if (is_acquire || is_volatile) { Node* n = _access.raw_access(); Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); @@ -212,6 +268,7 @@ } else if (is_write) { // If not multiple copy atomic, we do the MemBarVolatile before the load. if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) { + assert(kit != NULL, "unsupported at optimization time"); Node* n = _access.raw_access(); Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar if (_leading_membar != NULL) { @@ -220,6 +277,7 @@ } } else { if (is_volatile || is_acquire) { + assert(kit != NULL, "unsupported at optimization time"); Node* n = _access.raw_access(); assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected"); Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); @@ -295,7 +353,7 @@ if (!needs_cpu_membar() && adr_type->isa_instptr()) { assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null"); intptr_t offset = Type::OffsetBot; - AddPNode::Ideal_base_and_offset(adr, &_kit->gvn(), offset); + AddPNode::Ideal_base_and_offset(adr, &gvn(), offset); if (offset >= 0) { int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper()); if (offset < s) { @@ -310,26 +368,28 @@ //--------------------------- atomic operations--------------------------------- -void BarrierSetC2::pin_atomic_op(C2AtomicAccess& access) const { +void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const { if (!access.needs_pinning()) { return; } // SCMemProjNodes represent the memory state of a LoadStore. Their // main role is to prevent LoadStore nodes from being optimized away // when their results aren't used. - GraphKit* kit = access.kit(); + assert(access.is_parse_access(), "entry not supported at optimization time"); + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); Node* load_store = access.raw_access(); assert(load_store != NULL, "must pin atomic op"); Node* proj = kit->gvn().transform(new SCMemProjNode(load_store)); kit->set_memory(proj, access.alias_idx()); } -void C2AtomicAccess::set_memory() { +void C2AtomicParseAccess::set_memory() { Node *mem = _kit->memory(_alias_idx); _memory = mem; } -Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, +Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); MemNode::MemOrd mo = access.mem_node_mo(); @@ -386,7 +446,7 @@ return load_store; } -Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, +Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); DecoratorSet decorators = access.decorators(); @@ -460,7 +520,7 @@ return load_store; } -Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { +Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); Node* mem = access.memory(); Node* adr = access.addr().node(); @@ -508,7 +568,7 @@ return load_store; } -Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { +Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { Node* load_store = NULL; GraphKit* kit = access.kit(); Node* adr = access.addr().node(); @@ -538,27 +598,27 @@ return load_store; } -Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val, +Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { C2AccessFence fence(access); resolve_address(access); return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); } -Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val, +Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { C2AccessFence fence(access); resolve_address(access); return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); } -Node* BarrierSetC2::atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { +Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { C2AccessFence fence(access); resolve_address(access); return atomic_xchg_at_resolved(access, new_val, value_type); } -Node* BarrierSetC2::atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { +Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { C2AccessFence fence(access); resolve_address(access); return atomic_add_at_resolved(access, new_val, value_type); @@ -594,7 +654,7 @@ const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; - ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, false, false); + ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, true, false); ac->set_clonebasic(); Node* n = kit->gvn().transform(ac); if (n == ac) { @@ -731,3 +791,8 @@ } return fast_oop; } + +void BarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const { + // no barrier + igvn.replace_node(ac, call); +} --- old/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp 2018-10-26 11:08:20.988772224 +0200 +++ new/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp 2018-10-26 11:08:12.692750424 +0200 @@ -49,6 +49,10 @@ const DecoratorSet C2_WRITE_ACCESS = DECORATOR_LAST << 7; // This denotes that the access reads state. const DecoratorSet C2_READ_ACCESS = DECORATOR_LAST << 8; +// A nearby allocation? +const DecoratorSet C2_TIGHLY_COUPLED_ALLOC = DECORATOR_LAST << 9; +// Loads and stores from an arraycopy being optimized +const DecoratorSet C2_ARRAY_COPY = DECORATOR_LAST << 10; class GraphKit; class IdealKit; @@ -88,7 +92,6 @@ // BarrierSetC2 backend hierarchy, for loads and stores, to reduce boiler plate. class C2Access: public StackObj { protected: - GraphKit* _kit; DecoratorSet _decorators; BasicType _type; Node* _base; @@ -96,22 +99,17 @@ Node* _raw_access; void fixup_decorators(); - void* barrier_set_state() const; public: - C2Access(GraphKit* kit, DecoratorSet decorators, + C2Access(DecoratorSet decorators, BasicType type, Node* base, C2AccessValuePtr& addr) : - _kit(kit), _decorators(decorators), _type(type), _base(base), _addr(addr), _raw_access(NULL) - { - fixup_decorators(); - } + {} - GraphKit* kit() const { return _kit; } DecoratorSet decorators() const { return _decorators; } Node* base() const { return _base; } C2AccessValuePtr& addr() const { return _addr; } @@ -126,23 +124,48 @@ MemNode::MemOrd mem_node_mo() const; bool needs_cpu_membar() const; + virtual PhaseGVN& gvn() const = 0; + virtual bool is_parse_access() const { return false; } + virtual bool is_opt_access() const { return false; } +}; + +// C2Access for parse time calls to the BarrierSetC2 backend. +class C2ParseAccess: public C2Access { +protected: + GraphKit* _kit; + + void* barrier_set_state() const; + +public: + C2ParseAccess(GraphKit* kit, DecoratorSet decorators, + BasicType type, Node* base, C2AccessValuePtr& addr) : + C2Access(decorators, type, base, addr), + _kit(kit) { + fixup_decorators(); + } + + GraphKit* kit() const { return _kit; } + template T barrier_set_state_as() const { return reinterpret_cast(barrier_set_state()); } + + virtual PhaseGVN& gvn() const; + virtual bool is_parse_access() const { return true; } }; // This class wraps a bunch of context parameters thare are passed around in the // BarrierSetC2 backend hierarchy, for atomic accesses, to reduce boiler plate. -class C2AtomicAccess: public C2Access { +class C2AtomicParseAccess: public C2ParseAccess { Node* _memory; uint _alias_idx; bool _needs_pinning; public: - C2AtomicAccess(GraphKit* kit, DecoratorSet decorators, BasicType type, + C2AtomicParseAccess(GraphKit* kit, DecoratorSet decorators, BasicType type, Node* base, C2AccessValuePtr& addr, uint alias_idx) : - C2Access(kit, decorators, type, base, addr), + C2ParseAccess(kit, decorators, type, base, addr), _memory(NULL), _alias_idx(alias_idx), _needs_pinning(true) {} @@ -157,6 +180,31 @@ void set_needs_pinning(bool value) { _needs_pinning = value; } }; +// C2Access for optimization time calls to the BarrierSetC2 backend. +class C2OptAccess: public C2Access { + PhaseGVN& _gvn; + MergeMemNode* _mem; + Node* _ctl; + +public: + C2OptAccess(PhaseGVN& gvn, Node* ctl, MergeMemNode* mem, DecoratorSet decorators, + BasicType type, Node* base, C2AccessValuePtr& addr) : + C2Access(decorators, type, base, addr), + _gvn(gvn), _mem(mem), _ctl(ctl) { + fixup_decorators(); + } + + + MergeMemNode* mem() const { return _mem; } + Node* ctl() const { return _ctl; } + // void set_mem(Node* mem) { _mem = mem; } + void set_ctl(Node* ctl) { _ctl = ctl; } + + virtual PhaseGVN& gvn() const { return _gvn; } + virtual bool is_opt_access() const { return true; } +}; + + // This is the top-level class for the backend of the Access API in C2. // The top-level class is responsible for performing raw accesses. The // various GC barrier sets inherit from the BarrierSetC2 class to sprinkle @@ -167,25 +215,25 @@ virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const; virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; - virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, + virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const; - virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, + virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const; - virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const; - virtual Node* atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const; - void pin_atomic_op(C2AtomicAccess& access) const; + virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const; + virtual Node* atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const; + void pin_atomic_op(C2AtomicParseAccess& access) const; public: // This is the entry-point for the backend to perform accesses through the Access API. virtual Node* store_at(C2Access& access, C2AccessValue& val) const; virtual Node* load_at(C2Access& access, const Type* val_type) const; - virtual Node* atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val, + virtual Node* atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const; - virtual Node* atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val, + virtual Node* atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const; - virtual Node* atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const; - virtual Node* atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const; + virtual Node* atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const; + virtual Node* atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const; virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const; @@ -203,6 +251,7 @@ Expansion }; virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { return false; } + virtual void clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const; // Support for GC barriers emitted during parsing virtual bool has_load_barriers() const { return false; } --- old/src/hotspot/share/gc/shared/c2/modRefBarrierSetC2.cpp 2018-10-26 11:08:29.569795337 +0200 +++ new/src/hotspot/share/gc/shared/c2/modRefBarrierSetC2.cpp 2018-10-26 11:08:21.085772486 +0200 @@ -32,7 +32,6 @@ Node* ModRefBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { DecoratorSet decorators = access.decorators(); - GraphKit* kit = access.kit(); const TypePtr* adr_type = access.addr().type(); Node* adr = access.addr().node(); @@ -41,11 +40,16 @@ bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; bool in_heap = (decorators & IN_HEAP) != 0; bool use_precise = is_array || anonymous; + bool tighly_coupled_alloc = (decorators & C2_TIGHLY_COUPLED_ALLOC) != 0; - if (!access.is_oop() || (!in_heap && !anonymous)) { + if (!access.is_oop() || tighly_coupled_alloc || (!in_heap && !anonymous)) { return BarrierSetC2::store_at_resolved(access, val); } + assert(access.is_parse_access(), "entry not supported at optimization time"); + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); + uint adr_idx = kit->C->get_alias_index(adr_type); assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); @@ -58,7 +62,7 @@ return store; } -Node* ModRefBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, +Node* ModRefBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); @@ -78,7 +82,7 @@ return result; } -Node* ModRefBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, +Node* ModRefBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); @@ -114,7 +118,7 @@ return load_store; } -Node* ModRefBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { +Node* ModRefBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type); --- old/src/hotspot/share/gc/shared/c2/modRefBarrierSetC2.hpp 2018-10-26 11:08:38.077818254 +0200 +++ new/src/hotspot/share/gc/shared/c2/modRefBarrierSetC2.hpp 2018-10-26 11:08:29.662795588 +0200 @@ -54,11 +54,11 @@ virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const; - virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, + virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const; - virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, + virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const; - virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const; + virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const; }; #endif // SHARE_GC_SHARED_C2_MODREFBARRIERSETC2_HPP --- old/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp 2018-10-26 11:08:47.067842468 +0200 +++ new/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp 2018-10-26 11:08:38.171818507 +0200 @@ -474,10 +474,10 @@ // == Accesses == -Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const { +Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const { assert(!UseCompressedOops, "Not allowed"); CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access(); - PhaseGVN& gvn = access.kit()->gvn(); + PhaseGVN& gvn = access.gvn(); Compile* C = Compile::current(); GraphKit* kit = access.kit(); @@ -566,7 +566,7 @@ return phi; } -Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const { +Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const { CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access(); GraphKit* kit = access.kit(); PhaseGVN& gvn = kit->gvn(); @@ -665,7 +665,7 @@ } } -static bool barrier_needed(C2Access access) { +static bool barrier_needed(C2Access& access) { return ZBarrierSet::barrier_needed(access.decorators(), access.type()); } @@ -677,7 +677,9 @@ bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; - GraphKit* kit = access.kit(); + assert(access.is_parse_access(), "entry not supported at optimization time"); + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); PhaseGVN& gvn = kit->gvn(); Node* adr = access.addr().node(); Node* heap_base_oop = access.base(); @@ -707,11 +709,11 @@ } return p; } else { - return load_barrier(access.kit(), p, access.addr().node(), weak, true, true); + return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true); } } -Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, +Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const { Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); if (!barrier_needed(access)) { @@ -722,7 +724,7 @@ return make_cmpx_loadbarrier(access); } -Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, +Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); if (!barrier_needed(access)) { @@ -746,7 +748,7 @@ return load_store; } -Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { +Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const { Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); if (!barrier_needed(access)) { return result; @@ -755,7 +757,9 @@ Node* load_store = access.raw_access(); Node* adr = access.addr().node(); - return load_barrier(access.kit(), load_store, adr, false, false, false); + assert(access.is_parse_access(), "entry not supported at optimization time"); + C2ParseAccess& parse_access = static_cast(access); + return load_barrier(parse_access.kit(), load_store, adr, false, false, false); } // == Macro Expansion == --- old/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp 2018-10-26 11:08:55.724865786 +0200 +++ new/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp 2018-10-26 11:08:47.188842794 +0200 @@ -156,8 +156,8 @@ class ZBarrierSetC2 : public BarrierSetC2 { private: ZBarrierSetC2State* state() const; - Node* make_cas_loadbarrier(C2AtomicAccess& access) const; - Node* make_cmpx_loadbarrier(C2AtomicAccess& access) const; + Node* make_cas_loadbarrier(C2AtomicParseAccess& access) const; + Node* make_cmpx_loadbarrier(C2AtomicParseAccess& access) const; void expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const; void expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const; void expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const; @@ -165,15 +165,15 @@ protected: virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; - virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, + virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const; - virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, + virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const; - virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, + virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const; --- old/src/hotspot/share/opto/arraycopynode.cpp 2018-10-26 11:09:04.640889802 +0200 +++ new/src/hotspot/share/opto/arraycopynode.cpp 2018-10-26 11:08:55.830866072 +0200 @@ -148,6 +148,28 @@ return get_length_if_constant(phase); } +Node* ArrayCopyNode::load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, const Type *type, BasicType bt) { + DecoratorSet decorators = C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD | IN_HEAP | C2_ARRAY_COPY; + C2AccessValuePtr addr(adr, adr_type); + C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr); + Node* res = bs->load_at(access, type); + ctl = access.ctl(); + return res; +} + +void ArrayCopyNode::store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, Node* val, const Type *type, BasicType bt) { + DecoratorSet decorators = C2_WRITE_ACCESS | IN_HEAP | C2_ARRAY_COPY; + if (is_alloc_tightly_coupled()) { + decorators |= C2_TIGHLY_COUPLED_ALLOC; + } + C2AccessValuePtr addr(adr, adr_type); + C2AccessValue value(val, type); + C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr); + bs->store_at(access, value); + ctl = access.ctl(); +} + + Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) { if (!is_clonebasic()) { return NULL; @@ -182,6 +204,7 @@ ciInstanceKlass* ik = inst_src->klass()->as_instance_klass(); assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields"); + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); for (int i = 0; i < count; i++) { ciField* field = ik->nonstatic_field_at(i); int fieldidx = phase->C->alias_type(field)->index(); @@ -203,11 +226,8 @@ type = Type::get_const_basic_type(bt); } - Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered); - v = phase->transform(v); - Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered); - s = phase->transform(s); - mem->set_memory_at(fieldidx, s); + Node* v = load(bs, phase, ctl, mem, next_src, adr_type, type, bt); + store(bs, phase, ctl, mem, next_dest, adr_type, v, type, bt); } if (!finish_transform(phase, can_reshape, ctl, mem)) { @@ -368,28 +388,18 @@ if (!forward_ctl->is_top()) { // copy forward mm = mm->clone()->as_MergeMem(); - uint alias_idx_src = phase->C->get_alias_index(atp_src); - uint alias_idx_dest = phase->C->get_alias_index(atp_dest); - Node *start_mem_src = mm->memory_at(alias_idx_src); - Node *start_mem_dest = mm->memory_at(alias_idx_dest); - Node* mem = start_mem_dest; - bool same_alias = (alias_idx_src == alias_idx_dest); if (count > 0) { - Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered); - v = phase->transform(v); - mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered); - mem = phase->transform(mem); + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + Node* v = load(bs, phase, forward_ctl, mm, adr_src, atp_src, value_type, copy_type); + store(bs, phase, forward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type); for (int i = 1; i < count; i++) { Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); - v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered); - v = phase->transform(v); - mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered); - mem = phase->transform(mem); + v = load(bs, phase, forward_ctl, mm, next_src, atp_src, value_type, copy_type); + store(bs, phase, forward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type); } - mm->set_memory_at(alias_idx_dest, mem); } else if(can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); igvn->_worklist.push(adr_src); @@ -416,31 +426,20 @@ if (!backward_ctl->is_top()) { // copy backward mm = mm->clone()->as_MergeMem(); - uint alias_idx_src = phase->C->get_alias_index(atp_src); - uint alias_idx_dest = phase->C->get_alias_index(atp_dest); - Node *start_mem_src = mm->memory_at(alias_idx_src); - Node *start_mem_dest = mm->memory_at(alias_idx_dest); - Node* mem = start_mem_dest; BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays"); - bool same_alias = (alias_idx_src == alias_idx_dest); if (count > 0) { for (int i = count-1; i >= 1; i--) { Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); - Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered); - v = phase->transform(v); - mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered); - mem = phase->transform(mem); + Node* v = load(bs, phase, backward_ctl, mm, next_src, atp_src, value_type, copy_type); + store(bs, phase, backward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type); } - Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered); - v = phase->transform(v); - mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered); - mem = phase->transform(mem); - mm->set_memory_at(alias_idx_dest, mem); + Node* v = load(bs, phase, backward_ctl, mm, adr_src, atp_src, value_type, copy_type); + store(bs, phase, backward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type); } else if(can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); igvn->_worklist.push(adr_src); --- old/src/hotspot/share/opto/arraycopynode.hpp 2018-10-26 11:09:13.133912678 +0200 +++ new/src/hotspot/share/opto/arraycopynode.hpp 2018-10-26 11:09:04.748890093 +0200 @@ -110,6 +110,9 @@ Node* ctl, Node *mem); static bool may_modify_helper(const TypeOopPtr *t_oop, Node* n, PhaseTransform *phase, CallNode*& call); + static Node* load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* addr, const TypePtr* adr_type, const Type *type, BasicType bt); + void store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* addr, const TypePtr* adr_type, Node* val, const Type *type, BasicType bt); + public: enum { --- old/src/hotspot/share/opto/graphKit.cpp 2018-10-26 11:09:21.647935610 +0200 +++ new/src/hotspot/share/opto/graphKit.cpp 2018-10-26 11:09:13.225912925 +0200 @@ -1566,7 +1566,7 @@ C2AccessValuePtr addr(adr, adr_type); C2AccessValue value(val, val_type); - C2Access access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr); + C2ParseAccess access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr); if (access.is_raw()) { return _barrier_set->BarrierSetC2::store_at(access, value); } else { @@ -1585,7 +1585,7 @@ } C2AccessValuePtr addr(adr, adr_type); - C2Access access(this, decorators | C2_READ_ACCESS, bt, obj, addr); + C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, obj, addr); if (access.is_raw()) { return _barrier_set->BarrierSetC2::load_at(access, val_type); } else { @@ -1602,7 +1602,7 @@ } C2AccessValuePtr addr(adr, NULL); - C2Access access(this, decorators | C2_READ_ACCESS, bt, NULL, addr); + C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, NULL, addr); if (access.is_raw()) { return _barrier_set->BarrierSetC2::load_at(access, val_type); } else { @@ -1620,7 +1620,7 @@ BasicType bt, DecoratorSet decorators) { C2AccessValuePtr addr(adr, adr_type); - C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, + C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx); if (access.is_raw()) { return _barrier_set->BarrierSetC2::atomic_cmpxchg_val_at(access, expected_val, new_val, value_type); @@ -1639,7 +1639,7 @@ BasicType bt, DecoratorSet decorators) { C2AccessValuePtr addr(adr, adr_type); - C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, + C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx); if (access.is_raw()) { return _barrier_set->BarrierSetC2::atomic_cmpxchg_bool_at(access, expected_val, new_val, value_type); @@ -1657,7 +1657,7 @@ BasicType bt, DecoratorSet decorators) { C2AccessValuePtr addr(adr, adr_type); - C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, + C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx); if (access.is_raw()) { return _barrier_set->BarrierSetC2::atomic_xchg_at(access, new_val, value_type); @@ -1675,7 +1675,7 @@ BasicType bt, DecoratorSet decorators) { C2AccessValuePtr addr(adr, adr_type); - C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx); + C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx); if (access.is_raw()) { return _barrier_set->BarrierSetC2::atomic_add_at(access, new_val, value_type); } else { --- old/src/hotspot/share/opto/macro.cpp 2018-10-26 11:09:30.460958509 +0200 +++ new/src/hotspot/share/opto/macro.cpp 2018-10-26 11:09:21.769935939 +0200 @@ -940,6 +940,7 @@ } k -= (oc2 - use->outcnt()); } + _igvn.remove_dead_node(use); } else if (use->is_ArrayCopy()) { // Disconnect ArrayCopy node ArrayCopyNode* ac = use->as_ArrayCopy(); --- old/src/hotspot/share/opto/macroArrayCopy.cpp 2018-10-26 11:09:38.968980490 +0200 +++ new/src/hotspot/share/opto/macroArrayCopy.cpp 2018-10-26 11:09:30.566958783 +0200 @@ -72,7 +72,6 @@ Node* parm2, Node* parm3, Node* parm4, Node* parm5, Node* parm6, Node* parm7) { - int size = call_type->domain()->cnt(); Node* call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type); call->init_req(TypeFunc::Control, ctrl); call->init_req(TypeFunc::I_O , top()); @@ -1107,7 +1106,9 @@ Node* call = make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, src, dest, length XTOP); transform_later(call); - _igvn.replace_node(ac, call); + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + bs->clone_barrier_at_expansion(ac, call, _igvn); + return; } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) { Node* mem = ac->in(TypeFunc::Memory);