# HG changeset patch # Parent 1ca630d367179cae4c1231a7a0ad9ea61aac99dd diff -r 1ca630d36717 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Mon Dec 17 13:35:48 2018 +0100 @@ -400,30 +400,98 @@ #endif } +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, Register tmp) { +#ifdef _LP64 + Label done; + Label selfheal; + Label done_selfheal; + + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); + + Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); + __ jccb(Assembler::zero, done); + + // Move previous value into rax, we need it there for the wb-stub and the cmpxchg. + if (rax != dst) { + __ push(rax); + __ mov(rax, dst); + } + + // Move previous value into tmp, we need it for comparing with new-value. + __ mov(tmp, dst); + + // Heap is unstable, need to perform the read-barrier even if WB is inactive + read_barrier_not_null(masm, rax); + + __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); + __ jccb(Assembler::zero, selfheal); + + __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb()))); + + __ bind(selfheal); + + // Now we have forwarded value in rax, old value in scratch1 + __ cmpptr(rax, tmp); + __ jccb(Assembler::equal, done_selfheal); + __ xchgptr(rax, tmp); // Swap old value into rax (for cmpxchg) and new-value into scratch1 + __ cmpxchgptr(tmp, src); // Brings previous-value into rax (we don't need it) + __ xchgptr(rax, tmp); // Swap back forwarded-value into rax + __ bind(done_selfheal); + + // Now we have forwarded value in rax, old value in rscratch1 + if (rax != dst) { + __ movptr(dst, rax); + __ pop(rax); + } + __ bind(done); +#else + Unimplemented(); +#endif + +} + void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool in_heap = (decorators & IN_HEAP) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; - if (in_heap) { - read_barrier_not_null(masm, src.base()); - } - BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); - if (ShenandoahKeepAliveBarrier && on_oop && on_reference) { - const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); - NOT_LP64(__ get_thread(thread)); + if (on_oop) { + Register scratch1 = r8; + Register scratch2 = r9; + assert_different_registers(dst, scratch1, scratch2); + // Allocate scratch registers + __ push(scratch1); + __ push(scratch2); + + // Move src address into scratch2, it will be overridden by dst. + __ leaq(scratch2, src); + + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + load_reference_barrier(masm, dst, Address(scratch2, 0), scratch1); - // Generate the SATB pre-barrier code to log the value of - // the referent field in an SATB buffer. - shenandoah_write_barrier_pre(masm /* masm */, - noreg /* obj */, - dst /* pre_val */, - thread /* thread */, - tmp1 /* tmp */, - true /* tosca_live */, - true /* expand_call */); + // Restore scratch registers + __ pop(scratch2); + __ pop(scratch1); + + if (ShenandoahKeepAliveBarrier && on_reference) { + const Register thread = NOT_LP64(tmp_thread) + LP64_ONLY(r15_thread); + NOT_LP64(__ get_thread(thread)); + // Generate the SATB pre-barrier code to log the value of + // the referent field in an SATB buffer. + shenandoah_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + thread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + } + } else { + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); } } @@ -432,9 +500,6 @@ bool in_heap = (decorators & IN_HEAP) != 0; bool as_normal = (decorators & AS_NORMAL) != 0; - if (in_heap) { - write_barrier(masm, dst.base()); - } if (type == T_OBJECT || type == T_ARRAY) { bool needs_pre_barrier = as_normal; @@ -469,7 +534,6 @@ if (val == noreg) { BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg); } else { - storeval_barrier(masm, val, tmp3); BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg); } NOT_LP64(imasm->restore_bcp()); @@ -478,44 +542,6 @@ } } -#ifndef _LP64 -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, - Address obj1, jobject obj2) { - Unimplemented(); -} - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, - Register obj1, jobject obj2) { - Unimplemented(); -} -#endif - - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) { - __ cmpptr(op1, op2); - if (ShenandoahAcmpBarrier) { - Label done; - __ jccb(Assembler::equal, done); - read_barrier(masm, op1); - read_barrier(masm, op2); - __ cmpptr(op1, op2); - __ bind(done); - } -} - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) { - __ cmpptr(src1, src2); - if (ShenandoahAcmpBarrier) { - Label done; - __ jccb(Assembler::equal, done); - __ movptr(rscratch2, src2); - read_barrier(masm, src1); - read_barrier(masm, rscratch2); - __ cmpptr(src1, rscratch2); - __ bind(done); - } -} - void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, @@ -565,28 +591,6 @@ __ verify_tlab(); } -void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & IS_NOT_NULL) != 0; - bool is_write = (decorators & ACCESS_WRITE) != 0; - if (is_write) { - if (oop_not_null) { - write_barrier(masm, obj); - } else { - Label done; - __ testptr(obj, obj); - __ jcc(Assembler::zero, done); - write_barrier(masm, obj); - __ bind(done); - } - } else { - if (oop_not_null) { - read_barrier_not_null(masm, obj); - } else { - read_barrier(masm, obj); - } - } -} - // Special Shenandoah CAS implementation that handles false negatives // due to concurrent evacuation. #ifndef _LP64 @@ -724,6 +728,8 @@ assert(res != NULL, "need result register"); __ setb(Assembler::equal, res); __ movzbl(res, res); + } else { + write_barrier(masm, rax); } } #endif // LP64 diff -r 1ca630d36717 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Mon Dec 17 13:35:48 2018 +0100 @@ -81,6 +81,8 @@ void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); #endif + void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, Register tmp); + void cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, bool exchange, bool encode, Register tmp1, Register tmp2); @@ -93,16 +95,6 @@ virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2); -#ifndef _LP64 - virtual void obj_equals(MacroAssembler* masm, - Address obj1, jobject obj2); - virtual void obj_equals(MacroAssembler* masm, - Register obj1, jobject obj2); -#endif - - virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2); - virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2); - virtual void tlab_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, @@ -110,8 +102,6 @@ Register t1, Register t2, Label& slow_case); - virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj); - virtual void barrier_stubs_init(); }; diff -r 1ca630d36717 src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad Mon Dec 17 13:35:48 2018 +0100 @@ -25,6 +25,22 @@ #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" %} +instruct shenandoahLRB(rRegP dst, rRegP src, memory mem, rRegP tmp, rFlagsReg cr) %{ + match(Set dst (ShenandoahLoadReferenceBarrier src mem)); + effect(USE src, DEF dst, TEMP tmp, KILL cr); + ins_cost(125); // XXX + format %{ "shenandoah_lrb $src, $dst, $mem" %} + ins_encode %{ + Register s = $src$$Register; + Register d = $dst$$Register; + Register t = $tmp$$Register; + Address a = $mem$$Address; + __ movptr(d, s); + ShenandoahBarrierSet::assembler()->load_reference_barrier(&_masm, d, a, t); + %} + ins_pipe(pipe_slow); +%} + instruct shenandoahRB(rRegP dst, rRegP src, rFlagsReg cr) %{ match(Set dst (ShenandoahReadBarrier src)); effect(DEF dst, USE src); diff -r 1ca630d36717 src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Mon Dec 17 13:35:48 2018 +0100 @@ -548,6 +548,8 @@ } void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const { + BarrierSetC2::resolve_address(access); + /* const TypePtr* adr_type = access.addr().type(); if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) { @@ -627,6 +629,7 @@ } } } + */ } Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { @@ -648,9 +651,11 @@ uint adr_idx = kit->C->get_alias_index(adr_type); assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); + /* Node* value = val.node(); value = shenandoah_storeval_barrier(kit, value); val.set_node(value); + */ shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(), static_cast(val.type()), NULL /* pre_val */, access.type()); } else { @@ -722,6 +727,11 @@ Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top; Node* load = BarrierSetC2::load_at_resolved(access, val_type); + if (access.is_oop()) { + load = new ShenandoahLoadReferenceBarrierNode(NULL, load, adr); + load = static_cast(access).kit()->gvn().transform(load); + } + // If we are reading the value of the referent field of a Reference // object (either by using Unsafe directly or through reflection) // then, if SATB is enabled, we need to record the referent in an @@ -760,7 +770,7 @@ Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); if (access.is_oop()) { - new_val = shenandoah_storeval_barrier(kit, new_val); + // new_val = shenandoah_storeval_barrier(kit, new_val); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, expected_val /* pre_val */, T_OBJECT); @@ -787,9 +797,10 @@ #ifdef _LP64 if (adr->bottom_type()->is_ptr_to_narrowoop()) { - return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); } #endif + load_store = shenandoah_write_barrier(kit, load_store); return load_store; } return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); @@ -799,7 +810,7 @@ Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); if (access.is_oop()) { - new_val = shenandoah_storeval_barrier(kit, new_val); + // new_val = shenandoah_storeval_barrier(kit, new_val); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, expected_val /* pre_val */, T_OBJECT); @@ -836,9 +847,11 @@ Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const { GraphKit* kit = access.kit(); + /* if (access.is_oop()) { val = shenandoah_storeval_barrier(kit, val); } + */ Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type); if (access.is_oop()) { shenandoah_write_barrier_pre(kit, false /* do_load */, @@ -850,17 +863,20 @@ void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { assert(!src->is_AddP(), "unexpected input"); - src = shenandoah_read_barrier(kit, src); + // src = shenandoah_read_barrier(kit, src); BarrierSetC2::clone(kit, src, dst, size, is_array); } Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const { + return n; + /* bool is_write = decorators & ACCESS_WRITE; if (is_write) { return shenandoah_write_barrier(kit, n); } else { return shenandoah_read_barrier(kit, n); } + */ } Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, diff -r 1ca630d36717 src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp Mon Dec 17 13:35:48 2018 +0100 @@ -4420,3 +4420,24 @@ } } } + +ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, Node* addr) +: TypeNode(obj->bottom_type(), 3) { + + init_req(Control, ctrl); + init_req(ValueIn, obj); + init_req(Address, addr); +} + +const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const { + // Either input is TOP ==> the result is TOP + const Type *t2 = phase->type(in(ValueIn)); + if( t2 == Type::TOP ) return Type::TOP; + + if (t2 == TypePtr::NULL_PTR) { + return _type; + } + + const Type* type = t2->is_oopptr()->cast_to_nonconst(); + return type; +} diff -r 1ca630d36717 src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp Mon Dec 17 13:35:48 2018 +0100 @@ -382,4 +382,18 @@ virtual int Opcode() const; }; +class ShenandoahLoadReferenceBarrierNode : public TypeNode { +private: + enum { + Control, + ValueIn, + Address + }; +public: + ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val, Node* adr); + int Opcode() const; + virtual const Type* Value(PhaseGVN* phase) const; +}; + + #endif // SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAH_SUPPORT_HPP diff -r 1ca630d36717 src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Mon Dec 17 13:35:48 2018 +0100 @@ -102,6 +102,9 @@ bool obj_equals(oop obj1, oop obj2); + template + inline oop load_reference_barrier(T* r); + #ifdef CHECK_UNHANDLED_OOPS bool oop_equals_operator_allowed() { return !ShenandoahVerifyObjectEquals; } #endif @@ -150,76 +153,18 @@ typedef BarrierSet::AccessBarrier Raw; public: - // Primitive heap accesses. These accessors get resolved when - // IN_HEAP is set (e.g. when using the HeapAccess API), it is - // not an oop_* overload, and the barrier strength is AS_NORMAL. - template - static T load_in_heap(T* addr) { - ShouldNotReachHere(); - return Raw::template load(addr); - } - - template - static T load_in_heap_at(oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::resolve_forwarded(base); - return Raw::template load_at(base, offset); - } - - template - static void store_in_heap(T* addr, T value) { - ShouldNotReachHere(); - Raw::store(addr, value); - } - - template - static void store_in_heap_at(oop base, ptrdiff_t offset, T value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - Raw::store_at(base, offset, value); - } - - template - static T atomic_cmpxchg_in_heap(T new_value, T* addr, T compare_value) { - ShouldNotReachHere(); - return Raw::atomic_cmpxchg(new_value, addr, compare_value); - } - - template - static T atomic_cmpxchg_in_heap_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - return Raw::atomic_cmpxchg_at(new_value, base, offset, compare_value); - } - - template - static T atomic_xchg_in_heap(T new_value, T* addr) { - ShouldNotReachHere(); - return Raw::atomic_xchg(new_value, addr); - } - - template - static T atomic_xchg_in_heap_at(T new_value, oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - return Raw::atomic_xchg_at(new_value, base, offset); - } - - template - static void arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, - arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, - size_t length); - // Heap oop accesses. These accessors get resolved when // IN_HEAP is set (e.g. when using the HeapAccess API), it is // an oop_* overload, and the barrier strength is AS_NORMAL. template static oop oop_load_in_heap(T* addr) { - // ShouldNotReachHere(); - oop value = Raw::template oop_load(addr); + oop value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(addr); keep_alive_if_weak(decorators, value); return value; } static oop oop_load_in_heap_at(oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::resolve_forwarded(base); - oop value = Raw::template oop_load_at(base, offset); + oop value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(AccessInternal::oop_field_addr(base, offset)); keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); return value; } @@ -231,9 +176,6 @@ } static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); - oop_store_in_heap(AccessInternal::oop_field_addr(base, offset), value); } @@ -241,8 +183,6 @@ static oop oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value); static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr(base, offset), compare_value); } @@ -250,8 +190,6 @@ static oop oop_atomic_xchg_in_heap(oop new_value, T* addr); static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); return oop_atomic_xchg_in_heap(new_value, AccessInternal::oop_field_addr(base, offset)); } @@ -266,17 +204,19 @@ // Needed for loads on non-heap weak references template static oop oop_load_not_in_heap(T* addr) { - oop value = Raw::oop_load_not_in_heap(addr); + oop value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(addr); keep_alive_if_weak(decorators, value); return value; } - static oop resolve(oop obj) { - return ShenandoahBarrierSet::barrier_set()->write_barrier(obj); + template + static oop oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) { + return oop_atomic_cmpxchg_in_heap(new_value, addr, compare_value); } - static bool equals(oop o1, oop o2) { - return ShenandoahBarrierSet::barrier_set()->obj_equals(o1, o2); + template + static oop oop_atomic_xchg_not_in_heap(oop new_value, T* addr) { + return oop_atomic_xchg_in_heap(new_value, addr); } }; diff -r 1ca630d36717 src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Mon Dec 17 13:35:48 2018 +0100 @@ -47,6 +47,21 @@ } } +template +inline oop ShenandoahBarrierSet::load_reference_barrier(T* p) { + oop obj = RawAccess<>::oop_load(p); + if (obj != NULL) { + oop forw = write_barrier(obj); + if (!oopDesc::equals_raw(obj, forw)) { + RawAccess<>::oop_atomic_cmpxchg(forw, p, obj); + } + return forw; + } else { + return obj; + } + return obj; +} + template template inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { @@ -62,7 +77,11 @@ ShenandoahBarrierSet::barrier_set()->enqueue(compare_value); } } - return res; + if (res != NULL) { + return ShenandoahBarrierSet::barrier_set()->write_barrier(res); + } else { + return res; + } } template @@ -74,21 +93,11 @@ ShenandoahBarrierSet::barrier_set()->enqueue(previous); } } - return previous; -} - -template -template -void ShenandoahBarrierSet::AccessBarrier::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, - arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, - size_t length) { - if (!CompressedOops::is_null(src_obj)) { - src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj)); + if (previous != NULL) { + return ShenandoahBarrierSet::barrier_set()->write_barrier(previous); + } else { + return previous; } - if (!CompressedOops::is_null(dst_obj)) { - dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj)); - } - Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); } template diff -r 1ca630d36717 src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Mon Dec 17 13:35:48 2018 +0100 @@ -1036,10 +1036,10 @@ class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask { private: - ShenandoahRootEvacuator* _rp; + ShenandoahRootProcessor* _rp; public: - ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) : + ShenandoahEvacuateUpdateRootsTask(ShenandoahRootProcessor* rp) : AbstractGangTask("Shenandoah evacuate and update roots"), _rp(rp) {} @@ -1049,7 +1049,8 @@ ShenandoahEvacuateUpdateRootsClosure cl; MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations); - _rp->process_evacuate_roots(&cl, &blobsCl, worker_id); + CLDToOopClosure cldCl(&cl, ClassLoaderData::_claim_strong); + _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id); } }; @@ -1060,7 +1061,7 @@ assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped"); { - ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac); + ShenandoahRootProcessor rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac); ShenandoahEvacuateUpdateRootsTask roots_task(&rp); workers()->run_task(&roots_task); } diff -r 1ca630d36717 src/hotspot/share/opto/classes.hpp --- a/src/hotspot/share/opto/classes.hpp Fri Dec 14 15:33:42 2018 +0100 +++ b/src/hotspot/share/opto/classes.hpp Mon Dec 17 13:35:48 2018 +0100 @@ -280,6 +280,7 @@ shmacro(ShenandoahReadBarrier) shmacro(ShenandoahWriteBarrier) shmacro(ShenandoahWBMemProj) +shmacro(ShenandoahLoadReferenceBarrier) macro(SCMemProj) macro(SqrtD) macro(SqrtF)