# HG changeset patch # Parent 4c3faf49b391d0aabb2aa53604aedaf0dc8615d1 diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -42,368 +42,6 @@ #define __ masm-> -address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL; -address ShenandoahBarrierSetAssembler::_shenandoah_wb_C = NULL; - -void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, - Register src, Register dst, Register count) { - - bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; - bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; - bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); - bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; - - if (type == T_OBJECT || type == T_ARRAY) { -#ifdef _LP64 - if (!checkcast && !obj_int) { - // Save count for barrier - __ movptr(r11, count); - } else if (disjoint && obj_int) { - // Save dst in r11 in the disjoint case - __ movq(r11, dst); - } -#else - if (disjoint) { - __ mov(rdx, dst); // save 'to' - } -#endif - - if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { - Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); -#ifndef _LP64 - __ push(thread); - __ get_thread(thread); -#endif - - Label filtered; - Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); - // Is marking active? - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { - __ cmpl(in_progress, 0); - } else { - assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); - __ cmpb(in_progress, 0); - } - - NOT_LP64(__ pop(thread);) - - __ jcc(Assembler::equal, filtered); - - __ pusha(); // push registers -#ifdef _LP64 - if (count == c_rarg0) { - if (dst == c_rarg1) { - // exactly backwards!! - __ xchgptr(c_rarg1, c_rarg0); - } else { - __ movptr(c_rarg1, count); - __ movptr(c_rarg0, dst); - } - } else { - __ movptr(c_rarg0, dst); - __ movptr(c_rarg1, count); - } - if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2); - } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2); - } -#else - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), - dst, count); -#endif - __ popa(); - __ bind(filtered); - } - } - -} - -void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, - Register src, Register dst, Register count) { - bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; - bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; - bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); - Register tmp = rax; - - if (type == T_OBJECT || type == T_ARRAY) { -#ifdef _LP64 - if (!checkcast && !obj_int) { - // Save count for barrier - count = r11; - } else if (disjoint && obj_int) { - // Use the saved dst in the disjoint case - dst = r11; - } else if (checkcast) { - tmp = rscratch1; - } -#else - if (disjoint) { - __ mov(dst, rdx); // restore 'to' - } -#endif - - __ pusha(); // push registers (overkill) -#ifdef _LP64 - if (c_rarg0 == count) { // On win64 c_rarg0 == rcx - assert_different_registers(c_rarg1, dst); - __ mov(c_rarg1, count); - __ mov(c_rarg0, dst); - } else { - assert_different_registers(c_rarg0, count); - __ mov(c_rarg0, dst); - __ mov(c_rarg1, count); - } - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2); -#else - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), - dst, count); -#endif - __ popa(); - } -} - -void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, - Register obj, - Register pre_val, - Register thread, - Register tmp, - bool tosca_live, - bool expand_call) { - - if (ShenandoahSATBBarrier) { - satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call); - } -} - -void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, - Register obj, - Register pre_val, - Register thread, - Register tmp, - bool tosca_live, - bool expand_call) { - // If expand_call is true then we expand the call_VM_leaf macro - // directly to skip generating the check by - // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. - -#ifdef _LP64 - assert(thread == r15_thread, "must be"); -#endif // _LP64 - - Label done; - Label runtime; - - assert(pre_val != noreg, "check this code"); - - if (obj != noreg) { - assert_different_registers(obj, pre_val, tmp); - assert(pre_val != rax, "check this code"); - } - - Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); - Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); - Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); - - Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); - __ jcc(Assembler::zero, done); - - // Do we need to load the previous value? - if (obj != noreg) { - __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); - } - - // Is the previous value null? - __ cmpptr(pre_val, (int32_t) NULL_WORD); - __ jcc(Assembler::equal, done); - - // Can we store original value in the thread's buffer? - // Is index == 0? - // (The index field is typed as size_t.) - - __ movptr(tmp, index); // tmp := *index_adr - __ cmpptr(tmp, 0); // tmp == 0? - __ jcc(Assembler::equal, runtime); // If yes, goto runtime - - __ subptr(tmp, wordSize); // tmp := tmp - wordSize - __ movptr(index, tmp); // *index_adr := tmp - __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr - - // Record the previous value - __ movptr(Address(tmp, 0), pre_val); - __ jmp(done); - - __ bind(runtime); - // save the live input values - if(tosca_live) __ push(rax); - - if (obj != noreg && obj != rax) - __ push(obj); - - if (pre_val != rax) - __ push(pre_val); - - // Calling the runtime using the regular call_VM_leaf mechanism generates - // code (generated by InterpreterMacroAssember::call_VM_leaf_base) - // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL. - // - // If we care generating the pre-barrier without a frame (e.g. in the - // intrinsified Reference.get() routine) then ebp might be pointing to - // the caller frame and so this check will most likely fail at runtime. - // - // Expanding the call directly bypasses the generation of the check. - // So when we do not have have a full interpreter frame on the stack - // expand_call should be passed true. - - NOT_LP64( __ push(thread); ) - -#ifdef _LP64 - // We move pre_val into c_rarg0 early, in order to avoid smashing it, should - // pre_val be c_rarg1 (where the call prologue would copy thread argument). - // Note: this should not accidentally smash thread, because thread is always r15. - assert(thread != c_rarg0, "smashed arg"); - if (c_rarg0 != pre_val) { - __ mov(c_rarg0, pre_val); - } -#endif - - if (expand_call) { - LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); ) -#ifdef _LP64 - if (c_rarg1 != thread) { - __ mov(c_rarg1, thread); - } - // Already moved pre_val into c_rarg0 above -#else - __ push(thread); - __ push(pre_val); -#endif - __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2); - } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread); - } - - NOT_LP64( __ pop(thread); ) - - // save the live input values - if (pre_val != rax) - __ pop(pre_val); - - if (obj != noreg && obj != rax) - __ pop(obj); - - if(tosca_live) __ pop(rax); - - __ bind(done); -} - -void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); - Label is_null; - __ testptr(dst, dst); - __ jcc(Assembler::zero, is_null); - read_barrier_not_null_impl(masm, dst); - __ bind(is_null); -} - -void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_not_null_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); - __ movptr(dst, Address(dst, BrooksPointer::byte_offset())); -} - - -void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahWriteBarrier) { - write_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); -#ifdef _LP64 - Label done; - - Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); - __ jccb(Assembler::zero, done); - - // Heap is unstable, need to perform the read-barrier even if WB is inactive - if (ShenandoahWriteBarrierRB) { - read_barrier_not_null(masm, dst); - } - - __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); - __ jccb(Assembler::zero, done); - - if (dst != rax) { - __ xchgptr(dst, rax); // Move obj into rax and save rax into obj. - } - - __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb()))); - - if (dst != rax) { - __ xchgptr(rax, dst); // Swap back obj with rax. - } - - __ bind(done); -#else - Unimplemented(); -#endif -} - -void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { - if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) { - storeval_barrier_impl(masm, dst, tmp); - } -} - -void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { - assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled"); - - if (dst == noreg) return; - -#ifdef _LP64 - if (ShenandoahStoreValEnqueueBarrier) { - Label is_null; - __ testptr(dst, dst); - __ jcc(Assembler::zero, is_null); - write_barrier_impl(masm, dst); - __ bind(is_null); - - // The set of registers to be saved+restored is the same as in the write-barrier above. - // Those are the commonly used registers in the interpreter. - __ pusha(); - // __ push_callee_saved_registers(); - __ subptr(rsp, 2 * Interpreter::stackElementSize); - __ movdbl(Address(rsp, 0), xmm0); - - satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false); - __ movdbl(xmm0, Address(rsp, 0)); - __ addptr(rsp, 2 * Interpreter::stackElementSize); - //__ pop_callee_saved_registers(); - __ popa(); - } - if (ShenandoahStoreValReadBarrier) { - read_barrier_impl(masm, dst); - } -#else - Unimplemented(); -#endif -} - void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; @@ -416,7 +54,6 @@ } BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); if (ShenandoahKeepAliveBarrier && on_oop && on_reference) { - const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); NOT_LP64(__ get_thread(thread)); // Generate the SATB pre-barrier code to log the value of @@ -424,7 +61,7 @@ shenandoah_write_barrier_pre(masm /* masm */, noreg /* obj */, dst /* pre_val */, - thread /* thread */, + tmp_thread /* thread */, tmp1 /* tmp */, true /* tosca_live */, true /* expand_call */); @@ -520,55 +157,6 @@ } } -void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, - Register thread, Register obj, - Register var_size_in_bytes, - int con_size_in_bytes, - Register t1, Register t2, - Label& slow_case) { - assert_different_registers(obj, t1, t2); - assert_different_registers(obj, var_size_in_bytes, t1); - Register end = t2; - if (!thread->is_valid()) { -#ifdef _LP64 - thread = r15_thread; -#else - assert(t1->is_valid(), "need temp reg"); - thread = t1; - __ get_thread(thread); -#endif - } - - __ verify_tlab(); - - __ movptr(obj, Address(thread, JavaThread::tlab_top_offset())); - if (var_size_in_bytes == noreg) { - __ lea(end, Address(obj, con_size_in_bytes + BrooksPointer::byte_size())); - } else { - __ addptr(var_size_in_bytes, BrooksPointer::byte_size()); - __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); - } - __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); - __ jcc(Assembler::above, slow_case); - - // update the tlab top pointer - __ movptr(Address(thread, JavaThread::tlab_top_offset()), end); - - // Initialize brooks pointer -#ifdef _LP64 - __ incrementq(obj, BrooksPointer::byte_size()); -#else - __ incrementl(obj, BrooksPointer::byte_size()); -#endif - __ movptr(Address(obj, BrooksPointer::byte_offset()), obj); - - // recover var_size_in_bytes if necessary - if (var_size_in_bytes == end) { - __ subptr(var_size_in_bytes, obj); - } - __ verify_tlab(); -} - void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) { bool oop_not_null = (decorators & IS_NOT_NULL) != 0; bool is_write = (decorators & ACCESS_WRITE) != 0; @@ -591,384 +179,9 @@ } } -// Special Shenandoah CAS implementation that handles false negatives -// due to concurrent evacuation. -#ifndef _LP64 -void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, - Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2) { - // Shenandoah has no 32-bit version for this. - Unimplemented(); -} -#else -void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, - Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2) { - if (!ShenandoahCASBarrier) { -#ifdef _LP64 - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(oldval); - __ mov(rscratch1, newval); - __ encode_heap_oop(rscratch1); - newval = rscratch1; - } - if (os::is_MP()) { - __ lock(); - } - // oldval (rax) is implicitly used by this instruction - __ cmpxchgl(newval, addr); - } else -#endif - { - if (os::is_MP()) { - __ lock(); - } - __ cmpxchgptr(newval, addr); - } - - if (!exchange) { - assert(res != NULL, "need result register"); - __ setb(Assembler::equal, res); - __ movzbl(res, res); - } - return; - } - - assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled"); - assert(oldval == rax, "must be in rax for implicit use in cmpxchg"); - - Label retry, done; - - // Apply storeval barrier to newval. - if (encode) { - storeval_barrier(masm, newval, tmp1); - } - - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(oldval); - __ mov(rscratch1, newval); - __ encode_heap_oop(rscratch1); - newval = rscratch1; - } - } - - // Remember oldval for retry logic below - if (UseCompressedOops) { - __ movl(tmp1, oldval); - } else { - __ movptr(tmp1, oldval); - } - - // Step 1. Try to CAS with given arguments. If successful, then we are done, - // and can safely return. - if (os::is_MP()) __ lock(); - if (UseCompressedOops) { - __ cmpxchgl(newval, addr); - } else { - __ cmpxchgptr(newval, addr); - } - __ jcc(Assembler::equal, done, true); - - // Step 2. CAS had failed. This may be a false negative. - // - // The trouble comes when we compare the to-space pointer with the from-space - // pointer to the same object. To resolve this, it will suffice to read both - // oldval and the value from memory through the read barriers -- this will give - // both to-space pointers. If they mismatch, then it was a legitimate failure. - // - if (UseCompressedOops) { - __ decode_heap_oop(tmp1); - } - read_barrier_impl(masm, tmp1); - - if (UseCompressedOops) { - __ movl(tmp2, oldval); - __ decode_heap_oop(tmp2); - } else { - __ movptr(tmp2, oldval); - } - read_barrier_impl(masm, tmp2); - - __ cmpptr(tmp1, tmp2); - __ jcc(Assembler::notEqual, done, true); - - // Step 3. Try to CAS again with resolved to-space pointers. - // - // Corner case: it may happen that somebody stored the from-space pointer - // to memory while we were preparing for retry. Therefore, we can fail again - // on retry, and so need to do this in loop, always re-reading the failure - // witness through the read barrier. - __ bind(retry); - if (os::is_MP()) __ lock(); - if (UseCompressedOops) { - __ cmpxchgl(newval, addr); - } else { - __ cmpxchgptr(newval, addr); - } - __ jcc(Assembler::equal, done, true); - - if (UseCompressedOops) { - __ movl(tmp2, oldval); - __ decode_heap_oop(tmp2); - } else { - __ movptr(tmp2, oldval); - } - read_barrier_impl(masm, tmp2); - - __ cmpptr(tmp1, tmp2); - __ jcc(Assembler::equal, retry, true); - - // Step 4. If we need a boolean result out of CAS, check the flag again, - // and promote the result. Note that we handle the flag from both the CAS - // itself and from the retry loop. - __ bind(done); - if (!exchange) { - assert(res != NULL, "need result register"); - __ setb(Assembler::equal, res); - __ movzbl(res, res); - } -} -#endif // LP64 - void ShenandoahBarrierSetAssembler::xchg_oop(MacroAssembler* masm, DecoratorSet decorators, Register obj, Address addr, Register tmp) { storeval_barrier(masm, obj, tmp); BarrierSetAssembler::xchg_oop(masm, decorators, obj, addr, tmp); } -#ifdef COMPILER1 - -#undef __ -#define __ ce->masm()-> - -void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) { - ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); - // At this point we know that marking is in progress. - // If do_load() is true then we have to emit the - // load of the previous value; otherwise it has already - // been loaded into _pre_val. - - __ bind(*stub->entry()); - assert(stub->pre_val()->is_register(), "Precondition."); - - Register pre_val_reg = stub->pre_val()->as_register(); - - if (stub->do_load()) { - ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); - } - - __ cmpptr(pre_val_reg, (int32_t)NULL_WORD); - __ jcc(Assembler::equal, *stub->continuation()); - ce->store_parameter(stub->pre_val()->as_register(), 0); - __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); - __ jmp(*stub->continuation()); - -} - -void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) { - __ bind(*stub->entry()); - - Label done; - Register obj = stub->obj()->as_register(); - Register res = stub->result()->as_register(); - - if (res != obj) { - __ mov(res, obj); - } - - // Check for null. - if (stub->needs_null_check()) { - __ testptr(res, res); - __ jcc(Assembler::zero, done); - } - - write_barrier(ce->masm(), res); - - __ bind(done); - __ jmp(*stub->continuation()); -} - -#undef __ - -#define __ sasm-> - -void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { - __ prologue("shenandoah_pre_barrier", false); - // arg0 : previous value of memory - - __ push(rax); - __ push(rdx); - - const Register pre_val = rax; - const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); - const Register tmp = rdx; - - NOT_LP64(__ get_thread(thread);) - - Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); - Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); - - Label done; - Label runtime; - - // Is SATB still active? - Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); - __ jcc(Assembler::zero, done); - - // Can we store original value in the thread's buffer? - - __ movptr(tmp, queue_index); - __ testptr(tmp, tmp); - __ jcc(Assembler::zero, runtime); - __ subptr(tmp, wordSize); - __ movptr(queue_index, tmp); - __ addptr(tmp, buffer); - - // prev_val (rax) - __ load_parameter(0, pre_val); - __ movptr(Address(tmp, 0), pre_val); - __ jmp(done); - - __ bind(runtime); - - __ save_live_registers_no_oop_map(true); - - // load the pre-value - __ load_parameter(0, rcx); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread); - - __ restore_live_registers(true); - - __ bind(done); - - __ pop(rdx); - __ pop(rax); - - __ epilogue(); -} - -#undef __ - -#endif // COMPILER1 - -address ShenandoahBarrierSetAssembler::shenandoah_wb() { - assert(_shenandoah_wb != NULL, "need write barrier stub"); - return _shenandoah_wb; -} - -address ShenandoahBarrierSetAssembler::shenandoah_wb_C() { - assert(_shenandoah_wb_C != NULL, "need write barrier stub"); - return _shenandoah_wb_C; -} - -#define __ cgen->assembler()-> - -address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test) { - __ align(CodeEntryAlignment); - StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb"); - address start = __ pc(); - -#ifdef _LP64 - Label not_done; - - // We use RDI, which also serves as argument register for slow call. - // RAX always holds the src object ptr, except after the slow call and - // the cmpxchg, then it holds the result. - // R8 and RCX are used as temporary registers. - if (!c_abi) { - __ push(rdi); - __ push(r8); - } - - // Check for object beeing in the collection set. - // TODO: Can we use only 1 register here? - // The source object arrives here in rax. - // live: rax - // live: rdi - if (!c_abi) { - __ mov(rdi, rax); - } else { - if (rax != c_rarg0) { - __ mov(rax, c_rarg0); - } - } - if (do_cset_test) { - __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint()); - // live: r8 - __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); - __ movbool(r8, Address(r8, rdi, Address::times_1)); - // unlive: rdi - __ testbool(r8); - // unlive: r8 - __ jccb(Assembler::notZero, not_done); - - if (!c_abi) { - __ pop(r8); - __ pop(rdi); - } - __ ret(0); - - __ bind(not_done); - } - - if (!c_abi) { - __ push(rcx); - } - - if (!c_abi) { - __ push(rdx); - __ push(rdi); - __ push(rsi); - __ push(r8); - __ push(r9); - __ push(r10); - __ push(r11); - __ push(r12); - __ push(r13); - __ push(r14); - __ push(r15); - } - __ save_vector_registers(); - __ movptr(rdi, rax); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi); - __ restore_vector_registers(); - if (!c_abi) { - __ pop(r15); - __ pop(r14); - __ pop(r13); - __ pop(r12); - __ pop(r11); - __ pop(r10); - __ pop(r9); - __ pop(r8); - __ pop(rsi); - __ pop(rdi); - __ pop(rdx); - - __ pop(rcx); - __ pop(r8); - __ pop(rdi); - } - __ ret(0); -#else - ShouldNotReachHere(); -#endif - return start; -} - -#undef __ - -void ShenandoahBarrierSetAssembler::barrier_stubs_init() { - if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) { - int stub_code_size = 4096; - ResourceMark rm; - BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); - CodeBuffer buf(bb); - StubCodeGenerator cgen(&buf); - _shenandoah_wb = generate_shenandoah_wb(&cgen, false, true); - _shenandoah_wb_C = generate_shenandoah_wb(&cgen, true, !ShenandoahWriteBarrierCsetTestInIR); - } -} diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -25,7 +25,7 @@ #define CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP #include "asm/macroAssembler.hpp" -#include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp" #ifdef COMPILER1 class LIR_Assembler; class ShenandoahPreBarrierStub; @@ -34,64 +34,12 @@ class StubCodeGenerator; #endif -class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { -private: +class ShenandoahBarrierSetAssembler: public ShenandoahBaseBarrierSetAssembler { +public: - static address _shenandoah_wb; - static address _shenandoah_wb_C; - - void satb_write_barrier_pre(MacroAssembler* masm, - Register obj, - Register pre_val, - Register thread, - Register tmp, - bool tosca_live, - bool expand_call); - - void shenandoah_write_barrier_pre(MacroAssembler* masm, - Register obj, - Register pre_val, - Register thread, - Register tmp, - bool tosca_live, - bool expand_call); - - void read_barrier(MacroAssembler* masm, Register dst); - void read_barrier_impl(MacroAssembler* masm, Register dst); - - void read_barrier_not_null(MacroAssembler* masm, Register dst); - void read_barrier_not_null_impl(MacroAssembler* masm, Register dst); - - void write_barrier(MacroAssembler* masm, Register dst); - void write_barrier_impl(MacroAssembler* masm, Register dst); - - void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); - void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); - - address generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test); - -public: - static address shenandoah_wb(); - static address shenandoah_wb_C(); - - static bool is_shenandoah_wb_C_call(address call); - -#ifdef COMPILER1 - void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); - void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); - void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); -#endif - - void cmpxchg_oop(MacroAssembler* masm, - Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2); virtual void xchg_oop(MacroAssembler* masm, DecoratorSet decorators, Register obj, Address addr, Register tmp); - virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, - Register src, Register dst, Register count); - virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, - Register src, Register dst, Register count); virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, @@ -107,17 +55,8 @@ virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2); virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2); - virtual void tlab_allocate(MacroAssembler* masm, - Register thread, Register obj, - Register var_size_in_bytes, - int con_size_in_bytes, - Register t1, Register t2, - Label& slow_case); - virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj); - virtual void barrier_stubs_init(); - }; #endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_X86_HPP +#define CPU_X86_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_X86_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#ifdef COMPILER1 +class LIR_Assembler; +class ShenandoahPreBarrierStub; +class ShenandoahWriteBarrierStub; +class StubAssembler; +class StubCodeGenerator; +#endif + +class ShenandoahBaseBarrierSetAssembler: public BarrierSetAssembler { +private: + + static address _shenandoah_wb; + static address _shenandoah_wb_C; + + void satb_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call); + + void shenandoah_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call); + + void read_barrier(MacroAssembler* masm, Register dst); + void read_barrier_impl(MacroAssembler* masm, Register dst); + + void read_barrier_not_null(MacroAssembler* masm, Register dst); + void read_barrier_not_null_impl(MacroAssembler* masm, Register dst); + + void write_barrier(MacroAssembler* masm, Register dst); + void write_barrier_impl(MacroAssembler* masm, Register dst); + + void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); + void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); + + address generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test); + +public: + static address shenandoah_wb(); + static address shenandoah_wb_C(); + + static bool is_shenandoah_wb_C_call(address call); + +#ifdef COMPILER1 + void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); + void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); + void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); +#endif + + void cmpxchg_oop(MacroAssembler* masm, + Register res, Address addr, Register oldval, Register newval, + bool exchange, bool encode, Register tmp1, Register tmp2); + virtual void xchg_oop(MacroAssembler* masm, DecoratorSet decorators, + Register obj, Address addr, Register tmp); + + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count); + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count); + virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread); + virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); + +#ifndef _LP64 + virtual void obj_equals(MacroAssembler* masm, + Address obj1, jobject obj2); + virtual void obj_equals(MacroAssembler* masm, + Register obj1, jobject obj2); +#endif + + virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2); + virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2); + + virtual void tlab_allocate(MacroAssembler* masm, + Register thread, Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register t1, Register t2, + Label& slow_case); + + virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj); + + virtual void barrier_stubs_init(); + +}; + +#endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler_x86.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler_x86.cpp @@ -0,0 +1,837 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahHeuristics.hpp" +#include "gc/shenandoah/shenandoahRuntime.hpp" +#include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/interp_masm.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/thread.hpp" +#include "utilities/macros.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" +#endif + +#define __ masm-> + +address ShenandoahBaseBarrierSetAssembler::_shenandoah_wb = NULL; +address ShenandoahBaseBarrierSetAssembler::_shenandoah_wb_C = NULL; + +void ShenandoahBaseBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count) { + + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); + bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; + + if (type == T_OBJECT || type == T_ARRAY) { +#ifdef _LP64 + if (!checkcast && !obj_int) { + // Save count for barrier + __ movptr(r11, count); + } else if (disjoint && obj_int) { + // Save dst in r11 in the disjoint case + __ movq(r11, dst); + } +#else + if (disjoint) { + __ mov(rdx, dst); // save 'to' + } +#endif + + if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { + Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); +#ifndef _LP64 + __ push(thread); + __ get_thread(thread); +#endif + + Label filtered; + Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); + // Is marking active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + __ cmpl(in_progress, 0); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ cmpb(in_progress, 0); + } + + NOT_LP64(__ pop(thread);) + + __ jcc(Assembler::equal, filtered); + + __ pusha(); // push registers +#ifdef _LP64 + if (count == c_rarg0) { + if (dst == c_rarg1) { + // exactly backwards!! + __ xchgptr(c_rarg1, c_rarg0); + } else { + __ movptr(c_rarg1, count); + __ movptr(c_rarg0, dst); + } + } else { + __ movptr(c_rarg0, dst); + __ movptr(c_rarg1, count); + } + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2); + } +#else + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), + dst, count); +#endif + __ popa(); + __ bind(filtered); + } + } + +} + +void ShenandoahBaseBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count) { + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); + Register tmp = rax; + + if (type == T_OBJECT || type == T_ARRAY) { +#ifdef _LP64 + if (!checkcast && !obj_int) { + // Save count for barrier + count = r11; + } else if (disjoint && obj_int) { + // Use the saved dst in the disjoint case + dst = r11; + } else if (checkcast) { + tmp = rscratch1; + } +#else + if (disjoint) { + __ mov(dst, rdx); // restore 'to' + } +#endif + + __ pusha(); // push registers (overkill) +#ifdef _LP64 + if (c_rarg0 == count) { // On win64 c_rarg0 == rcx + assert_different_registers(c_rarg1, dst); + __ mov(c_rarg1, count); + __ mov(c_rarg0, dst); + } else { + assert_different_registers(c_rarg0, count); + __ mov(c_rarg0, dst); + __ mov(c_rarg1, count); + } + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2); +#else + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), + dst, count); +#endif + __ popa(); + } +} + +void ShenandoahBaseBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call) { + + if (ShenandoahSATBBarrier) { + satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call); + } +} + +void ShenandoahBaseBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call) { + // If expand_call is true then we expand the call_VM_leaf macro + // directly to skip generating the check by + // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. + +#ifdef _LP64 + assert(thread == r15_thread, "must be"); +#endif // _LP64 + + Label done; + Label runtime; + + assert(pre_val != noreg, "check this code"); + + if (obj != noreg) { + assert_different_registers(obj, pre_val, tmp); + assert(pre_val != rax, "check this code"); + } + + Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); + Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); + Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); + + Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); + __ jcc(Assembler::zero, done); + + // Do we need to load the previous value? + if (obj != noreg) { + __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); + } + + // Is the previous value null? + __ cmpptr(pre_val, (int32_t) NULL_WORD); + __ jcc(Assembler::equal, done); + + // Can we store original value in the thread's buffer? + // Is index == 0? + // (The index field is typed as size_t.) + + __ movptr(tmp, index); // tmp := *index_adr + __ cmpptr(tmp, 0); // tmp == 0? + __ jcc(Assembler::equal, runtime); // If yes, goto runtime + + __ subptr(tmp, wordSize); // tmp := tmp - wordSize + __ movptr(index, tmp); // *index_adr := tmp + __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr + + // Record the previous value + __ movptr(Address(tmp, 0), pre_val); + __ jmp(done); + + __ bind(runtime); + // save the live input values + if(tosca_live) __ push(rax); + + if (obj != noreg && obj != rax) + __ push(obj); + + if (pre_val != rax) + __ push(pre_val); + + // Calling the runtime using the regular call_VM_leaf mechanism generates + // code (generated by InterpreterMacroAssember::call_VM_leaf_base) + // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL. + // + // If we care generating the pre-barrier without a frame (e.g. in the + // intrinsified Reference.get() routine) then ebp might be pointing to + // the caller frame and so this check will most likely fail at runtime. + // + // Expanding the call directly bypasses the generation of the check. + // So when we do not have have a full interpreter frame on the stack + // expand_call should be passed true. + + NOT_LP64( __ push(thread); ) + +#ifdef _LP64 + // We move pre_val into c_rarg0 early, in order to avoid smashing it, should + // pre_val be c_rarg1 (where the call prologue would copy thread argument). + // Note: this should not accidentally smash thread, because thread is always r15. + assert(thread != c_rarg0, "smashed arg"); + if (c_rarg0 != pre_val) { + __ mov(c_rarg0, pre_val); + } +#endif + + if (expand_call) { + LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); ) +#ifdef _LP64 + if (c_rarg1 != thread) { + __ mov(c_rarg1, thread); + } + // Already moved pre_val into c_rarg0 above +#else + __ push(thread); + __ push(pre_val); +#endif + __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread); + } + + NOT_LP64( __ pop(thread); ) + + // save the live input values + if (pre_val != rax) + __ pop(pre_val); + + if (obj != noreg && obj != rax) + __ pop(obj); + + if(tosca_live) __ pop(rax); + + __ bind(done); +} + +void ShenandoahBaseBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) { + if (ShenandoahReadBarrier) { + read_barrier_impl(masm, dst); + } +} + +void ShenandoahBaseBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) { + assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); + Label is_null; + __ testptr(dst, dst); + __ jcc(Assembler::zero, is_null); + read_barrier_not_null_impl(masm, dst); + __ bind(is_null); +} + +void ShenandoahBaseBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) { + if (ShenandoahReadBarrier) { + read_barrier_not_null_impl(masm, dst); + } +} + +void ShenandoahBaseBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) { + assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); + __ movptr(dst, Address(dst, BrooksPointer::byte_offset())); +} + + +void ShenandoahBaseBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) { + if (ShenandoahWriteBarrier) { + write_barrier_impl(masm, dst); + } +} + +void ShenandoahBaseBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) { + assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); +#ifdef _LP64 + Label done; + + Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); + __ jccb(Assembler::zero, done); + + // Heap is unstable, need to perform the read-barrier even if WB is inactive + if (ShenandoahWriteBarrierRB) { + read_barrier_not_null(masm, dst); + } + + __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); + __ jccb(Assembler::zero, done); + + if (dst != rax) { + __ xchgptr(dst, rax); // Move obj into rax and save rax into obj. + } + + __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBaseBarrierSetAssembler::shenandoah_wb()))); + + if (dst != rax) { + __ xchgptr(rax, dst); // Swap back obj with rax. + } + + __ bind(done); +#else + Unimplemented(); +#endif +} + +void ShenandoahBaseBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { + if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) { + storeval_barrier_impl(masm, dst, tmp); + } +} + +void ShenandoahBaseBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { + assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled"); + + if (dst == noreg) return; + +#ifdef _LP64 + if (ShenandoahStoreValEnqueueBarrier) { + Label is_null; + __ testptr(dst, dst); + __ jcc(Assembler::zero, is_null); + write_barrier_impl(masm, dst); + __ bind(is_null); + + enqueue_barrier(masm, dst, tmp); + } + if (ShenandoahStoreValReadBarrier) { + read_barrier_impl(masm, dst); + } +#else + Unimplemented(); +#endif +} + +void ShenandoahBaseBarrierSetAssembler::enqueue_barrier(MacroAssembler* masm, Register dst, Register tmp) { + if (ShenandoahStoreValEnqueueBarrier) { + // The set of registers to be saved+restored is the same as in the write-barrier above. + // Those are the commonly used registers in the interpreter. + __ pusha(); + // __ push_callee_saved_registers(); + __ subptr(rsp, 2 * Interpreter::stackElementSize); + __ movdbl(Address(rsp, 0), xmm0); + + satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false); + __ movdbl(xmm0, Address(rsp, 0)); + __ addptr(rsp, 2 * Interpreter::stackElementSize); + //__ pop_callee_saved_registers(); + __ popa(); + } +} + +void ShenandoahBaseBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, + Register thread, Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register t1, Register t2, + Label& slow_case) { + assert_different_registers(obj, t1, t2); + assert_different_registers(obj, var_size_in_bytes, t1); + Register end = t2; + if (!thread->is_valid()) { +#ifdef _LP64 + thread = r15_thread; +#else + assert(t1->is_valid(), "need temp reg"); + thread = t1; + __ get_thread(thread); +#endif + } + + __ verify_tlab(); + + __ movptr(obj, Address(thread, JavaThread::tlab_top_offset())); + if (var_size_in_bytes == noreg) { + __ lea(end, Address(obj, con_size_in_bytes + BrooksPointer::byte_size())); + } else { + __ addptr(var_size_in_bytes, BrooksPointer::byte_size()); + __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); + } + __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); + __ jcc(Assembler::above, slow_case); + + // update the tlab top pointer + __ movptr(Address(thread, JavaThread::tlab_top_offset()), end); + + // Initialize brooks pointer +#ifdef _LP64 + __ incrementq(obj, BrooksPointer::byte_size()); +#else + __ incrementl(obj, BrooksPointer::byte_size()); +#endif + __ movptr(Address(obj, BrooksPointer::byte_offset()), obj); + + // recover var_size_in_bytes if necessary + if (var_size_in_bytes == end) { + __ subptr(var_size_in_bytes, obj); + } + __ verify_tlab(); +} + + +// Special Shenandoah CAS implementation that handles false negatives +// due to concurrent evacuation. +#ifndef _LP64 +void ShenandoahBaseBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, + Register res, Address addr, Register oldval, Register newval, + bool exchange, bool encode, Register tmp1, Register tmp2) { + // Shenandoah has no 32-bit version for this. + Unimplemented(); +} +#else +void ShenandoahBaseBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, + Register res, Address addr, Register oldval, Register newval, + bool exchange, bool encode, Register tmp1, Register tmp2) { + if (!ShenandoahCASBarrier) { +#ifdef _LP64 + if (UseCompressedOops) { + if (encode) { + __ encode_heap_oop(oldval); + __ mov(rscratch1, newval); + __ encode_heap_oop(rscratch1); + newval = rscratch1; + } + if (os::is_MP()) { + __ lock(); + } + // oldval (rax) is implicitly used by this instruction + __ cmpxchgl(newval, addr); + } else +#endif + { + if (os::is_MP()) { + __ lock(); + } + __ cmpxchgptr(newval, addr); + } + + if (!exchange) { + assert(res != NULL, "need result register"); + __ setb(Assembler::equal, res); + __ movzbl(res, res); + } + return; + } + + assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled"); + assert(oldval == rax, "must be in rax for implicit use in cmpxchg"); + + Label retry, done; + + // Apply storeval barrier to newval. + if (encode) { + storeval_barrier(masm, newval, tmp1); + } + + if (UseCompressedOops) { + if (encode) { + __ encode_heap_oop(oldval); + __ mov(rscratch1, newval); + __ encode_heap_oop(rscratch1); + newval = rscratch1; + } + } + + // Remember oldval for retry logic below + if (UseCompressedOops) { + __ movl(tmp1, oldval); + } else { + __ movptr(tmp1, oldval); + } + + // Step 1. Try to CAS with given arguments. If successful, then we are done, + // and can safely return. + if (os::is_MP()) __ lock(); + if (UseCompressedOops) { + __ cmpxchgl(newval, addr); + } else { + __ cmpxchgptr(newval, addr); + } + __ jcc(Assembler::equal, done, true); + + // Step 2. CAS had failed. This may be a false negative. + // + // The trouble comes when we compare the to-space pointer with the from-space + // pointer to the same object. To resolve this, it will suffice to read both + // oldval and the value from memory through the read barriers -- this will give + // both to-space pointers. If they mismatch, then it was a legitimate failure. + // + if (UseCompressedOops) { + __ decode_heap_oop(tmp1); + } + read_barrier_impl(masm, tmp1); + + if (UseCompressedOops) { + __ movl(tmp2, oldval); + __ decode_heap_oop(tmp2); + } else { + __ movptr(tmp2, oldval); + } + read_barrier_impl(masm, tmp2); + + __ cmpptr(tmp1, tmp2); + __ jcc(Assembler::notEqual, done, true); + + // Step 3. Try to CAS again with resolved to-space pointers. + // + // Corner case: it may happen that somebody stored the from-space pointer + // to memory while we were preparing for retry. Therefore, we can fail again + // on retry, and so need to do this in loop, always re-reading the failure + // witness through the read barrier. + __ bind(retry); + if (os::is_MP()) __ lock(); + if (UseCompressedOops) { + __ cmpxchgl(newval, addr); + } else { + __ cmpxchgptr(newval, addr); + } + __ jcc(Assembler::equal, done, true); + + if (UseCompressedOops) { + __ movl(tmp2, oldval); + __ decode_heap_oop(tmp2); + } else { + __ movptr(tmp2, oldval); + } + read_barrier_impl(masm, tmp2); + + __ cmpptr(tmp1, tmp2); + __ jcc(Assembler::equal, retry, true); + + // Step 4. If we need a boolean result out of CAS, check the flag again, + // and promote the result. Note that we handle the flag from both the CAS + // itself and from the retry loop. + __ bind(done); + if (!exchange) { + assert(res != NULL, "need result register"); + __ setb(Assembler::equal, res); + __ movzbl(res, res); + } +} +#endif // LP64 + +#ifdef COMPILER1 + +#undef __ +#define __ ce->masm()-> + +void ShenandoahBaseBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) { + ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + // At this point we know that marking is in progress. + // If do_load() is true then we have to emit the + // load of the previous value; otherwise it has already + // been loaded into _pre_val. + + __ bind(*stub->entry()); + assert(stub->pre_val()->is_register(), "Precondition."); + + Register pre_val_reg = stub->pre_val()->as_register(); + + if (stub->do_load()) { + ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); + } + + __ cmpptr(pre_val_reg, (int32_t)NULL_WORD); + __ jcc(Assembler::equal, *stub->continuation()); + ce->store_parameter(stub->pre_val()->as_register(), 0); + __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); + __ jmp(*stub->continuation()); + +} + +void ShenandoahBaseBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) { + __ bind(*stub->entry()); + + Label done; + Register obj = stub->obj()->as_register(); + Register res = stub->result()->as_register(); + + if (res != obj) { + __ mov(res, obj); + } + + // Check for null. + if (stub->needs_null_check()) { + __ testptr(res, res); + __ jcc(Assembler::zero, done); + } + + write_barrier(ce->masm(), res); + + __ bind(done); + __ jmp(*stub->continuation()); +} + +#undef __ + +#define __ sasm-> + +void ShenandoahBaseBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("shenandoah_pre_barrier", false); + // arg0 : previous value of memory + + __ push(rax); + __ push(rdx); + + const Register pre_val = rax; + const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); + const Register tmp = rdx; + + NOT_LP64(__ get_thread(thread);) + + Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); + Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); + + Label done; + Label runtime; + + // Is SATB still active? + Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); + __ jcc(Assembler::zero, done); + + // Can we store original value in the thread's buffer? + + __ movptr(tmp, queue_index); + __ testptr(tmp, tmp); + __ jcc(Assembler::zero, runtime); + __ subptr(tmp, wordSize); + __ movptr(queue_index, tmp); + __ addptr(tmp, buffer); + + // prev_val (rax) + __ load_parameter(0, pre_val); + __ movptr(Address(tmp, 0), pre_val); + __ jmp(done); + + __ bind(runtime); + + __ save_live_registers_no_oop_map(true); + + // load the pre-value + __ load_parameter(0, rcx); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread); + + __ restore_live_registers(true); + + __ bind(done); + + __ pop(rdx); + __ pop(rax); + + __ epilogue(); +} + +#undef __ + +#endif // COMPILER1 + +address ShenandoahBaseBarrierSetAssembler::shenandoah_wb() { + assert(_shenandoah_wb != NULL, "need write barrier stub"); + return _shenandoah_wb; +} + +address ShenandoahBaseBarrierSetAssembler::shenandoah_wb_C() { + assert(_shenandoah_wb_C != NULL, "need write barrier stub"); + return _shenandoah_wb_C; +} + +#define __ cgen->assembler()-> + +address ShenandoahBaseBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test) { + __ align(CodeEntryAlignment); + StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb"); + address start = __ pc(); + +#ifdef _LP64 + Label not_done; + + // We use RDI, which also serves as argument register for slow call. + // RAX always holds the src object ptr, except after the slow call and + // the cmpxchg, then it holds the result. + // R8 and RCX are used as temporary registers. + if (!c_abi) { + __ push(rdi); + __ push(r8); + } + + // Check for object beeing in the collection set. + // TODO: Can we use only 1 register here? + // The source object arrives here in rax. + // live: rax + // live: rdi + if (!c_abi) { + __ mov(rdi, rax); + } else { + if (rax != c_rarg0) { + __ mov(rax, c_rarg0); + } + } + if (do_cset_test) { + __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + // live: r8 + __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + __ movbool(r8, Address(r8, rdi, Address::times_1)); + // unlive: rdi + __ testbool(r8); + // unlive: r8 + __ jccb(Assembler::notZero, not_done); + + if (!c_abi) { + __ pop(r8); + __ pop(rdi); + } + __ ret(0); + + __ bind(not_done); + } + + if (!c_abi) { + __ push(rcx); + } + + if (!c_abi) { + __ push(rdx); + __ push(rdi); + __ push(rsi); + __ push(r8); + __ push(r9); + __ push(r10); + __ push(r11); + __ push(r12); + __ push(r13); + __ push(r14); + __ push(r15); + } + __ save_vector_registers(); + __ movptr(rdi, rax); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi); + __ restore_vector_registers(); + if (!c_abi) { + __ pop(r15); + __ pop(r14); + __ pop(r13); + __ pop(r12); + __ pop(r11); + __ pop(r10); + __ pop(r9); + __ pop(r8); + __ pop(rsi); + __ pop(rdi); + __ pop(rdx); + + __ pop(rcx); + __ pop(r8); + __ pop(rdi); + } + __ ret(0); +#else + ShouldNotReachHere(); +#endif + return start; +} + +#undef __ + +void ShenandoahBaseBarrierSetAssembler::barrier_stubs_init() { + if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) { + int stub_code_size = 4096; + ResourceMark rm; + BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); + CodeBuffer buf(bb); + StubCodeGenerator cgen(&buf); + _shenandoah_wb = generate_shenandoah_wb(&cgen, false, true); + _shenandoah_wb_C = generate_shenandoah_wb(&cgen, true, !ShenandoahWriteBarrierCsetTestInIR); + } +} diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler_x86.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBaseBarrierSetAssembler_x86.hpp @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_X86_HPP +#define CPU_X86_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_X86_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#ifdef COMPILER1 +class LIR_Assembler; +class ShenandoahPreBarrierStub; +class ShenandoahWriteBarrierStub; +class StubAssembler; +class StubCodeGenerator; +#endif + +class ShenandoahBaseBarrierSetAssembler: public BarrierSetAssembler { +private: + + static address _shenandoah_wb; + static address _shenandoah_wb_C; + + void read_barrier_not_null_impl(MacroAssembler* masm, Register dst); + void write_barrier_impl(MacroAssembler* masm, Register dst); + void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); + +protected: + void satb_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call); + + void shenandoah_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call); + + void read_barrier(MacroAssembler* masm, Register dst); + void read_barrier_not_null(MacroAssembler* masm, Register dst); + void read_barrier_impl(MacroAssembler* masm, Register dst); + + void write_barrier(MacroAssembler* masm, Register dst); + + void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); + void enqueue_barrier(MacroAssembler* masm, Register dst, Register tmp); + + address generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test); + +public: + static address shenandoah_wb(); + static address shenandoah_wb_C(); + + static bool is_shenandoah_wb_C_call(address call); + + void cmpxchg_oop(MacroAssembler* masm, + Register res, Address addr, Register oldval, Register newval, + bool exchange, bool encode, Register tmp1, Register tmp2); + +#ifdef COMPILER1 + void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); + void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); + void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); +#endif + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count); + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count); + + virtual void tlab_allocate(MacroAssembler* masm, + Register thread, Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register t1, Register t2, + Label& slow_case); + + virtual void barrier_stubs_init(); +}; + +#endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahLRBBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahLRBBarrierSetAssembler_x86.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahLRBBarrierSetAssembler_x86.cpp @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahLRBBarrierSetAssembler.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/interp_masm.hpp" + +#define __ masm-> + +void ShenandoahLRBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread) { + + bool on_oop = type == T_OBJECT || type == T_ARRAY; + bool in_heap = (decorators & IN_HEAP) != 0; + bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; + bool on_reference = on_weak || on_phantom; + ShenandoahBaseBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + if (on_oop) { + Label is_null; + __ testptr(dst, dst); + __ jcc(Assembler::zero, is_null); + write_barrier(masm, dst); + __ bind(is_null); + } + if (ShenandoahKeepAliveBarrier && on_oop && on_reference) { + const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); + NOT_LP64(__ get_thread(thread)); + + // Generate the SATB pre-barrier code to log the value of + // the referent field in an SATB buffer. + shenandoah_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + thread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + } +} + +void ShenandoahLRBBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + + bool in_heap = (decorators & IN_HEAP) != 0; + bool as_normal = (decorators & AS_NORMAL) != 0; + if (type == T_OBJECT || type == T_ARRAY) { + bool needs_pre_barrier = as_normal; + + Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi); + Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx); + // flatten object address if needed + // We do it regardless of precise because we need the registers + if (dst.index() == noreg && dst.disp() == 0) { + if (dst.base() != tmp1) { + __ movptr(tmp1, dst.base()); + } + } else { + __ lea(tmp1, dst); + } + +#ifndef _LP64 + InterpreterMacroAssembler *imasm = static_cast(masm); +#endif + + NOT_LP64(__ get_thread(rcx)); + NOT_LP64(imasm->save_bcp()); + + if (needs_pre_barrier) { + shenandoah_write_barrier_pre(masm /*masm*/, + tmp1 /* obj */, + tmp2 /* pre_val */, + rthread /* thread */, + tmp3 /* tmp */, + val != noreg /* tosca_live */, + false /* expand_call */); + } + if (val == noreg) { + BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg); + } else { + enqueue_barrier(masm, val, tmp3); + BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg); + } + NOT_LP64(imasm->restore_bcp()); + } else { + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); + } +} diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahLRBBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahLRBBarrierSetAssembler_x86.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahLRBBarrierSetAssembler_x86.hpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSETASSEMBLER_X86_HPP +#define CPU_X86_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSETASSEMBLER_X86_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp" + +class ShenandoahLRBBarrierSetAssembler: public ShenandoahBaseBarrierSetAssembler { +public: + virtual void load_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread); + + virtual void store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); +}; + +#endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/share/gc/shared/barrierSetConfig.hpp b/src/hotspot/share/gc/shared/barrierSetConfig.hpp --- a/src/hotspot/share/gc/shared/barrierSetConfig.hpp +++ b/src/hotspot/share/gc/shared/barrierSetConfig.hpp @@ -33,10 +33,12 @@ EPSILONGC_ONLY(f(EpsilonBarrierSet)) \ G1GC_ONLY(f(G1BarrierSet)) \ ZGC_ONLY(f(ZBarrierSet)) \ - SHENANDOAHGC_ONLY(f(Shenandoah)) + SHENANDOAHGC_ONLY(f(Shenandoah)) \ + SHENANDOAHGC_ONLY(f(ShenandoahLRB)) #define FOR_EACH_ABSTRACT_BARRIER_SET_DO(f) \ - f(ModRef) + f(ModRef) \ + SHENANDOAHGC_ONLY(f(ShenandoahBase)) // Do something for each known barrier set. #define FOR_EACH_BARRIER_SET_DO(f) \ diff --git a/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp b/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp --- a/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp +++ b/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp @@ -40,7 +40,9 @@ #include "gc/z/zBarrierSet.inline.hpp" #endif #if INCLUDE_SHENANDOAHGC -#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" // Shenandoah support +#include "gc/shenandoah/shenandoahBaseBarrierSet.inline.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" +#include "gc/shenandoah/shenandoahLRBBarrierSet.inline.hpp" #endif #endif // SHARE_VM_GC_SHARED_BARRIERSETCONFIG_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -25,7 +25,7 @@ #include "c1/c1_IR.hpp" #include "gc/shared/satbMarkQueue.hpp" #include "gc/shenandoah/brooksPointer.hpp" -#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" @@ -37,166 +37,6 @@ #define __ gen->lir()-> #endif -void ShenandoahPreBarrierStub::emit_code(LIR_Assembler* ce) { - ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - bs->gen_pre_barrier_stub(ce, this); -} - -void ShenandoahWriteBarrierStub::emit_code(LIR_Assembler* ce) { - ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - bs->gen_write_barrier_stub(ce, this); -} - -void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) { - - // First we test whether marking is in progress. - BasicType flag_type; - bool patch = (decorators & C1_NEEDS_PATCHING) != 0; - bool do_load = pre_val == LIR_OprFact::illegalOpr; - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { - flag_type = T_INT; - } else { - guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, - "Assumption"); - // Use unsigned type T_BOOLEAN here rather than signed T_BYTE since some platforms, eg. ARM, - // need to use unsigned instructions to use the large offset to load the satb_mark_queue. - flag_type = T_BOOLEAN; - } - LIR_Opr thrd = gen->getThreadPointer(); - LIR_Address* mark_active_flag_addr = - new LIR_Address(thrd, - in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()), - flag_type); - // Read the marking-in-progress flag. - LIR_Opr flag_val = gen->new_register(T_INT); - __ load(mark_active_flag_addr, flag_val); - __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); - - LIR_PatchCode pre_val_patch_code = lir_patch_none; - - CodeStub* slow; - - if (do_load) { - assert(pre_val == LIR_OprFact::illegalOpr, "sanity"); - assert(addr_opr != LIR_OprFact::illegalOpr, "sanity"); - - if (patch) - pre_val_patch_code = lir_patch_normal; - - pre_val = gen->new_register(T_OBJECT); - - if (!addr_opr->is_address()) { - assert(addr_opr->is_register(), "must be"); - addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, T_OBJECT)); - } - slow = new ShenandoahPreBarrierStub(addr_opr, pre_val, pre_val_patch_code, info ? new CodeEmitInfo(info) : NULL); - } else { - assert(addr_opr == LIR_OprFact::illegalOpr, "sanity"); - assert(pre_val->is_register(), "must be"); - assert(pre_val->type() == T_OBJECT, "must be an object"); - - slow = new ShenandoahPreBarrierStub(pre_val); - } - - __ branch(lir_cond_notEqual, T_INT, slow); - __ branch_destination(slow->continuation()); -} - -LIR_Opr ShenandoahBarrierSetC1::read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - if (UseShenandoahGC && ShenandoahReadBarrier) { - return read_barrier_impl(gen, obj, info, need_null_check); - } else { - return obj; - } -} - -LIR_Opr ShenandoahBarrierSetC1::read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "Should be enabled"); - LabelObj* done = new LabelObj(); - LIR_Opr result = gen->new_register(T_OBJECT); - __ move(obj, result); - if (need_null_check) { - __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL)); - __ branch(lir_cond_equal, T_LONG, done->label()); - } - LIR_Address* brooks_ptr_address = gen->generate_address(result, BrooksPointer::byte_offset(), T_ADDRESS); - __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none); - - __ branch_destination(done->label()); - return result; -} - -LIR_Opr ShenandoahBarrierSetC1::write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - if (UseShenandoahGC && ShenandoahWriteBarrier) { - return write_barrier_impl(gen, obj, info, need_null_check); - } else { - return obj; - } -} - -LIR_Opr ShenandoahBarrierSetC1::write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); - - obj = ensure_in_register(gen, obj); - assert(obj->is_register(), "must be a register at this point"); - LIR_Opr result = gen->new_register(T_OBJECT); - __ move(obj, result); - - LIR_Opr thrd = gen->getThreadPointer(); - LIR_Address* active_flag_addr = - new LIR_Address(thrd, - in_bytes(ShenandoahThreadLocalData::gc_state_offset()), - T_BYTE); - // Read and check the gc-state-flag. - LIR_Opr flag_val = gen->new_register(T_INT); - __ load(active_flag_addr, flag_val); - LIR_Opr mask = LIR_OprFact::intConst(ShenandoahHeap::HAS_FORWARDED | - ShenandoahHeap::EVACUATION | - ShenandoahHeap::TRAVERSAL); - LIR_Opr mask_reg = gen->new_register(T_INT); - __ move(mask, mask_reg); - - if (TwoOperandLIRForm) { - __ logical_and(flag_val, mask_reg, flag_val); - } else { - LIR_Opr masked_flag = gen->new_register(T_INT); - __ logical_and(flag_val, mask_reg, masked_flag); - flag_val = masked_flag; - } - __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); - - CodeStub* slow = new ShenandoahWriteBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check); - __ branch(lir_cond_notEqual, T_INT, slow); - __ branch_destination(slow->continuation()); - - return result; -} - -LIR_Opr ShenandoahBarrierSetC1::ensure_in_register(LIRGenerator* gen, LIR_Opr obj) { - if (!obj->is_register()) { - LIR_Opr obj_reg = gen->new_register(T_OBJECT); - if (obj->is_constant()) { - __ move(obj, obj_reg); - } else { - __ leal(obj, obj_reg); - } - obj = obj_reg; - } - return obj; -} - -LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) { - bool need_null_check = (decorators & IS_NOT_NULL) == 0; - if (ShenandoahStoreValEnqueueBarrier) { - obj = write_barrier_impl(gen, obj, info, need_null_check); - pre_barrier(gen, info, decorators, LIR_OprFact::illegalOpr, obj); - } - if (ShenandoahStoreValReadBarrier) { - obj = read_barrier_impl(gen, obj, info, true /*need_null_check*/); - } - return obj; -} - void ShenandoahBarrierSetC1::store_at(LIRAccess& access, LIR_Opr value) { access.set_base(write_barrier(access.gen(), access.base().item().result(), access.access_emit_info(), access.needs_null_check())); LIR_Opr resolved = resolve_address(access, false); @@ -283,19 +123,3 @@ return read_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0); } } - -class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { - virtual OopMapSet* generate_code(StubAssembler* sasm) { - ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_pre_barrier_runtime_stub(sasm); - return NULL; - } -}; - -void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { - C1ShenandoahPreBarrierCodeGenClosure pre_code_gen_cl; - _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, - "shenandoah_pre_barrier_slow", - false, &pre_code_gen_cl); -} - diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp @@ -24,189 +24,17 @@ #ifndef SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP #define SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP -#include "c1/c1_CodeStubs.hpp" -#include "gc/shared/c1/barrierSetC1.hpp" +#include "gc/shenandoah/c1/shenandoahBaseBarrierSetC1.hpp" - -class ShenandoahPreBarrierStub: public CodeStub { - friend class ShenandoahBarrierSetC1; - private: - bool _do_load; - LIR_Opr _addr; - LIR_Opr _pre_val; - LIR_PatchCode _patch_code; - CodeEmitInfo* _info; - - public: - // Version that _does_ generate a load of the previous value from addr. - // addr (the address of the field to be read) must be a LIR_Address - // pre_val (a temporary register) must be a register; - ShenandoahPreBarrierStub(LIR_Opr addr, LIR_Opr pre_val, LIR_PatchCode patch_code, CodeEmitInfo* info) : - _do_load(true), _addr(addr), _pre_val(pre_val), - _patch_code(patch_code), _info(info) - { - assert(_pre_val->is_register(), "should be temporary register"); - assert(_addr->is_address(), "should be the address of the field"); - } - - // Version that _does not_ generate load of the previous value; the - // previous value is assumed to have already been loaded into pre_val. - ShenandoahPreBarrierStub(LIR_Opr pre_val) : - _do_load(false), _addr(LIR_OprFact::illegalOpr), _pre_val(pre_val), - _patch_code(lir_patch_none), _info(NULL) - { - assert(_pre_val->is_register(), "should be a register"); - } - - LIR_Opr addr() const { return _addr; } - LIR_Opr pre_val() const { return _pre_val; } - LIR_PatchCode patch_code() const { return _patch_code; } - CodeEmitInfo* info() const { return _info; } - bool do_load() const { return _do_load; } - - virtual void emit_code(LIR_Assembler* e); - virtual void visit(LIR_OpVisitState* visitor) { - if (_do_load) { - // don't pass in the code emit info since it's processed in the fast - // path - if (_info != NULL) - visitor->do_slow_case(_info); - else - visitor->do_slow_case(); - - visitor->do_input(_addr); - visitor->do_temp(_pre_val); - } else { - visitor->do_slow_case(); - visitor->do_input(_pre_val); - } - } -#ifndef PRODUCT - virtual void print_name(outputStream* out) const { out->print("ShenandoahPreBarrierStub"); } -#endif // PRODUCT -}; - -class ShenandoahWriteBarrierStub: public CodeStub { - friend class ShenandoahBarrierSetC1; - private: - LIR_Opr _obj; - LIR_Opr _result; - CodeEmitInfo* _info; - bool _needs_null_check; - - public: - ShenandoahWriteBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) : - _obj(obj), _result(result), _info(info), _needs_null_check(needs_null_check) - { - assert(_obj->is_register(), "should be register"); - assert(_result->is_register(), "should be register"); - } - - LIR_Opr obj() const { return _obj; } - LIR_Opr result() const { return _result; } - CodeEmitInfo* info() const { return _info; } - bool needs_null_check() const { return _needs_null_check; } - - virtual void emit_code(LIR_Assembler* e); - virtual void visit(LIR_OpVisitState* visitor) { - visitor->do_slow_case(); - visitor->do_input(_obj); - visitor->do_temp(_result); - } -#ifndef PRODUCT - virtual void print_name(outputStream* out) const { out->print("ShenandoahWritePreBarrierStub"); } -#endif // PRODUCT -}; - -class LIR_OpShenandoahCompareAndSwap : public LIR_Op { - friend class LIR_OpVisitState; - - private: - LIR_Opr _addr; - LIR_Opr _cmp_value; - LIR_Opr _new_value; - LIR_Opr _tmp1; - LIR_Opr _tmp2; - - public: - LIR_OpShenandoahCompareAndSwap(LIR_Opr addr, LIR_Opr cmp_value, LIR_Opr new_value, - LIR_Opr t1, LIR_Opr t2, LIR_Opr result) - : LIR_Op(lir_none, result, NULL) // no info - , _addr(addr) - , _cmp_value(cmp_value) - , _new_value(new_value) - , _tmp1(t1) - , _tmp2(t2) { } - - LIR_Opr addr() const { return _addr; } - LIR_Opr cmp_value() const { return _cmp_value; } - LIR_Opr new_value() const { return _new_value; } - LIR_Opr tmp1() const { return _tmp1; } - LIR_Opr tmp2() const { return _tmp2; } - - virtual void visit(LIR_OpVisitState* state) { - assert(_addr->is_valid(), "used"); - assert(_cmp_value->is_valid(), "used"); - assert(_new_value->is_valid(), "used"); - if (_info) state->do_info(_info); - state->do_input(_addr); - state->do_temp(_addr); - state->do_input(_cmp_value); - state->do_temp(_cmp_value); - state->do_input(_new_value); - state->do_temp(_new_value); - if (_tmp1->is_valid()) state->do_temp(_tmp1); - if (_tmp2->is_valid()) state->do_temp(_tmp2); - if (_result->is_valid()) state->do_output(_result); - } - - virtual void emit_code(LIR_Assembler* masm); - - virtual void print_instr(outputStream* out) const { - addr()->print(out); out->print(" "); - cmp_value()->print(out); out->print(" "); - new_value()->print(out); out->print(" "); - tmp1()->print(out); out->print(" "); - tmp2()->print(out); out->print(" "); - } -#ifndef PRODUCT - virtual const char* name() const { - return "shenandoah_cas_obj"; - } -#endif // PRODUCT -}; - -class ShenandoahBarrierSetC1 : public BarrierSetC1 { -private: - CodeBlob* _pre_barrier_c1_runtime_code_blob; - - void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val); - - LIR_Opr read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators); - - LIR_Opr read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - - LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj); - - virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value); - +class ShenandoahBarrierSetC1 : public ShenandoahBaseBarrierSetC1 { public: - CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; } - - virtual void store_at(LIRAccess& access, LIR_Opr value); - virtual void load_at(LIRAccess& access, LIR_Opr result); - - virtual LIR_Opr atomic_cmpxchg_at(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value); - - virtual LIR_Opr atomic_xchg_at(LIRAccess& access, LIRItem& value); - virtual LIR_Opr atomic_add_at(LIRAccess& access, LIRItem& value); - - virtual LIR_Opr resolve(LIRGenerator* gen, DecoratorSet decorators, LIR_Opr obj); - - virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob); + virtual void store_at(LIRAccess &access, LIR_Opr value); + virtual void load_at(LIRAccess &access, LIR_Opr result); + virtual LIR_Opr atomic_cmpxchg_at(LIRAccess &access, LIRItem &cmp_value, LIRItem &new_value); + virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess &access, LIRItem &cmp_value, LIRItem &new_value); + virtual LIR_Opr atomic_xchg_at(LIRAccess &access, LIRItem &value); + virtual LIR_Opr atomic_add_at(LIRAccess &access, LIRItem &value); + virtual LIR_Opr resolve(LIRGenerator *gen, DecoratorSet decorators, LIR_Opr obj); }; #endif // SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBaseBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBaseBarrierSetC1.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBaseBarrierSetC1.cpp @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "c1/c1_IR.hpp" +#include "gc/shared/satbMarkQueue.hpp" +#include "gc/shenandoah/brooksPointer.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "gc/shenandoah/c1/shenandoahBaseBarrierSetC1.hpp" + +#ifdef ASSERT +#define __ gen->lir(__FILE__, __LINE__)-> +#else +#define __ gen->lir()-> +#endif + +void ShenandoahPreBarrierStub::emit_code(LIR_Assembler* ce) { + ShenandoahBaseBarrierSetAssembler* bs = (ShenandoahBaseBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + bs->gen_pre_barrier_stub(ce, this); +} + +void ShenandoahWriteBarrierStub::emit_code(LIR_Assembler* ce) { + ShenandoahBaseBarrierSetAssembler* bs = (ShenandoahBaseBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + bs->gen_write_barrier_stub(ce, this); +} + +void ShenandoahBaseBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) { + + // First we test whether marking is in progress. + BasicType flag_type; + bool patch = (decorators & C1_NEEDS_PATCHING) != 0; + bool do_load = pre_val == LIR_OprFact::illegalOpr; + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + flag_type = T_INT; + } else { + guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, + "Assumption"); + // Use unsigned type T_BOOLEAN here rather than signed T_BYTE since some platforms, eg. ARM, + // need to use unsigned instructions to use the large offset to load the satb_mark_queue. + flag_type = T_BOOLEAN; + } + LIR_Opr thrd = gen->getThreadPointer(); + LIR_Address* mark_active_flag_addr = + new LIR_Address(thrd, + in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()), + flag_type); + // Read the marking-in-progress flag. + LIR_Opr flag_val = gen->new_register(T_INT); + __ load(mark_active_flag_addr, flag_val); + __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); + + LIR_PatchCode pre_val_patch_code = lir_patch_none; + + CodeStub* slow; + + if (do_load) { + assert(pre_val == LIR_OprFact::illegalOpr, "sanity"); + assert(addr_opr != LIR_OprFact::illegalOpr, "sanity"); + + if (patch) + pre_val_patch_code = lir_patch_normal; + + pre_val = gen->new_register(T_OBJECT); + + if (!addr_opr->is_address()) { + assert(addr_opr->is_register(), "must be"); + addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, T_OBJECT)); + } + slow = new ShenandoahPreBarrierStub(addr_opr, pre_val, pre_val_patch_code, info ? new CodeEmitInfo(info) : NULL); + } else { + assert(addr_opr == LIR_OprFact::illegalOpr, "sanity"); + assert(pre_val->is_register(), "must be"); + assert(pre_val->type() == T_OBJECT, "must be an object"); + + slow = new ShenandoahPreBarrierStub(pre_val); + } + + __ branch(lir_cond_notEqual, T_INT, slow); + __ branch_destination(slow->continuation()); +} + +LIR_Opr ShenandoahBaseBarrierSetC1::read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + if (UseShenandoahGC && ShenandoahReadBarrier) { + return read_barrier_impl(gen, obj, info, need_null_check); + } else { + return obj; + } +} + +LIR_Opr ShenandoahBaseBarrierSetC1::read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "Should be enabled"); + LabelObj* done = new LabelObj(); + LIR_Opr result = gen->new_register(T_OBJECT); + __ move(obj, result); + if (need_null_check) { + __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL)); + __ branch(lir_cond_equal, T_LONG, done->label()); + } + LIR_Address* brooks_ptr_address = gen->generate_address(result, BrooksPointer::byte_offset(), T_ADDRESS); + __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none); + + __ branch_destination(done->label()); + return result; +} + +LIR_Opr ShenandoahBaseBarrierSetC1::write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + if (UseShenandoahGC && ShenandoahWriteBarrier) { + return write_barrier_impl(gen, obj, info, need_null_check); + } else { + return obj; + } +} + +LIR_Opr ShenandoahBaseBarrierSetC1::write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); + + obj = ensure_in_register(gen, obj); + assert(obj->is_register(), "must be a register at this point"); + LIR_Opr result = gen->new_register(T_OBJECT); + __ move(obj, result); + + LIR_Opr thrd = gen->getThreadPointer(); + LIR_Address* active_flag_addr = + new LIR_Address(thrd, + in_bytes(ShenandoahThreadLocalData::gc_state_offset()), + T_BYTE); + // Read and check the gc-state-flag. + LIR_Opr flag_val = gen->new_register(T_INT); + __ load(active_flag_addr, flag_val); + LIR_Opr mask = LIR_OprFact::intConst(ShenandoahHeap::HAS_FORWARDED | + ShenandoahHeap::EVACUATION | + ShenandoahHeap::TRAVERSAL); + LIR_Opr mask_reg = gen->new_register(T_INT); + __ move(mask, mask_reg); + + if (TwoOperandLIRForm) { + __ logical_and(flag_val, mask_reg, flag_val); + } else { + LIR_Opr masked_flag = gen->new_register(T_INT); + __ logical_and(flag_val, mask_reg, masked_flag); + flag_val = masked_flag; + } + __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); + + CodeStub* slow = new ShenandoahWriteBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check); + __ branch(lir_cond_notEqual, T_INT, slow); + __ branch_destination(slow->continuation()); + + return result; +} + +LIR_Opr ShenandoahBaseBarrierSetC1::ensure_in_register(LIRGenerator* gen, LIR_Opr obj) { + if (!obj->is_register()) { + LIR_Opr obj_reg = gen->new_register(T_OBJECT); + if (obj->is_constant()) { + __ move(obj, obj_reg); + } else { + __ leal(obj, obj_reg); + } + obj = obj_reg; + } + return obj; +} + +LIR_Opr ShenandoahBaseBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) { + bool need_null_check = (decorators & IS_NOT_NULL) == 0; + if (ShenandoahStoreValEnqueueBarrier) { + obj = write_barrier_impl(gen, obj, info, need_null_check); + enqueue_barrier(gen, obj, info, decorators); + } + if (ShenandoahStoreValReadBarrier) { + obj = read_barrier_impl(gen, obj, info, true /*need_null_check*/); + } + return obj; +} + +void ShenandoahBaseBarrierSetC1::enqueue_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) { + if (ShenandoahStoreValEnqueueBarrier) { + obj = ensure_in_register(gen, obj); + pre_barrier(gen, info, decorators, LIR_OprFact::illegalOpr, obj); + } +} + +class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { + virtual OopMapSet* generate_code(StubAssembler* sasm) { + ShenandoahBaseBarrierSetAssembler* bs = (ShenandoahBaseBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + bs->generate_c1_pre_barrier_runtime_stub(sasm); + return NULL; + } +}; + +void ShenandoahBaseBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { + C1ShenandoahPreBarrierCodeGenClosure pre_code_gen_cl; + _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, + "shenandoah_pre_barrier_slow", + false, &pre_code_gen_cl); +} diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBaseBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBaseBarrierSetC1.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBaseBarrierSetC1.hpp @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBASEBARRIERSETC1_HPP +#define SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBASEBARRIERSETC1_HPP + +#include "c1/c1_CodeStubs.hpp" +#include "gc/shared/c1/barrierSetC1.hpp" + +class ShenandoahPreBarrierStub: public CodeStub { + friend class ShenandoahBaseBarrierSetC1; +private: + bool _do_load; + LIR_Opr _addr; + LIR_Opr _pre_val; + LIR_PatchCode _patch_code; + CodeEmitInfo* _info; + +public: + // Version that _does_ generate a load of the previous value from addr. + // addr (the address of the field to be read) must be a LIR_Address + // pre_val (a temporary register) must be a register; + ShenandoahPreBarrierStub(LIR_Opr addr, LIR_Opr pre_val, LIR_PatchCode patch_code, CodeEmitInfo* info) : + _do_load(true), _addr(addr), _pre_val(pre_val), + _patch_code(patch_code), _info(info) + { + assert(_pre_val->is_register(), "should be temporary register"); + assert(_addr->is_address(), "should be the address of the field"); + } + + // Version that _does not_ generate load of the previous value; the + // previous value is assumed to have already been loaded into pre_val. + ShenandoahPreBarrierStub(LIR_Opr pre_val) : + _do_load(false), _addr(LIR_OprFact::illegalOpr), _pre_val(pre_val), + _patch_code(lir_patch_none), _info(NULL) + { + assert(_pre_val->is_register(), "should be a register"); + } + + LIR_Opr addr() const { return _addr; } + LIR_Opr pre_val() const { return _pre_val; } + LIR_PatchCode patch_code() const { return _patch_code; } + CodeEmitInfo* info() const { return _info; } + bool do_load() const { return _do_load; } + + virtual void emit_code(LIR_Assembler* e); + virtual void visit(LIR_OpVisitState* visitor) { + if (_do_load) { + // don't pass in the code emit info since it's processed in the fast + // path + if (_info != NULL) + visitor->do_slow_case(_info); + else + visitor->do_slow_case(); + visitor->do_input(_addr); + visitor->do_temp(_pre_val); + } else { + visitor->do_slow_case(); + visitor->do_input(_pre_val); + } + } +#ifndef PRODUCT + virtual void print_name(outputStream* out) const { out->print("ShenandoahPreBarrierStub"); } +#endif // PRODUCT +}; + +class ShenandoahWriteBarrierStub: public CodeStub { + friend class ShenandoahBaseBarrierSetC1; +private: + LIR_Opr _obj; + LIR_Opr _result; + CodeEmitInfo* _info; + bool _needs_null_check; + +public: + ShenandoahWriteBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) : + _obj(obj), _result(result), _info(info), _needs_null_check(needs_null_check) + { + assert(_obj->is_register(), "should be register"); + assert(_result->is_register(), "should be register"); + } + + LIR_Opr obj() const { return _obj; } + LIR_Opr result() const { return _result; } + CodeEmitInfo* info() const { return _info; } + bool needs_null_check() const { return _needs_null_check; } + + virtual void emit_code(LIR_Assembler* e); + virtual void visit(LIR_OpVisitState* visitor) { + visitor->do_slow_case(); + visitor->do_input(_obj); + visitor->do_temp(_result); + } +#ifndef PRODUCT + virtual void print_name(outputStream* out) const { out->print("ShenandoahWritePreBarrierStub"); } +#endif // PRODUCT +}; + +class LIR_OpShenandoahCompareAndSwap : public LIR_Op { + friend class LIR_OpVisitState; + +private: + LIR_Opr _addr; + LIR_Opr _cmp_value; + LIR_Opr _new_value; + LIR_Opr _tmp1; + LIR_Opr _tmp2; + +public: + LIR_OpShenandoahCompareAndSwap(LIR_Opr addr, LIR_Opr cmp_value, LIR_Opr new_value, + LIR_Opr t1, LIR_Opr t2, LIR_Opr result) + : LIR_Op(lir_none, result, NULL) // no info + , _addr(addr) + , _cmp_value(cmp_value) + , _new_value(new_value) + , _tmp1(t1) + , _tmp2(t2) { } + + LIR_Opr addr() const { return _addr; } + LIR_Opr cmp_value() const { return _cmp_value; } + LIR_Opr new_value() const { return _new_value; } + LIR_Opr tmp1() const { return _tmp1; } + LIR_Opr tmp2() const { return _tmp2; } + + virtual void visit(LIR_OpVisitState* state) { + assert(_addr->is_valid(), "used"); + assert(_cmp_value->is_valid(), "used"); + assert(_new_value->is_valid(), "used"); + if (_info) state->do_info(_info); + state->do_input(_addr); + state->do_temp(_addr); + state->do_input(_cmp_value); + state->do_temp(_cmp_value); + state->do_input(_new_value); + state->do_temp(_new_value); + if (_tmp1->is_valid()) state->do_temp(_tmp1); + if (_tmp2->is_valid()) state->do_temp(_tmp2); + if (_result->is_valid()) state->do_output(_result); + } + + virtual void emit_code(LIR_Assembler* masm); + + virtual void print_instr(outputStream* out) const { + addr()->print(out); out->print(" "); + cmp_value()->print(out); out->print(" "); + new_value()->print(out); out->print(" "); + tmp1()->print(out); out->print(" "); + tmp2()->print(out); out->print(" "); + } +#ifndef PRODUCT + virtual const char* name() const { + return "shenandoah_cas_obj"; + } +#endif // PRODUCT +}; + +class ShenandoahBaseBarrierSetC1 : public BarrierSetC1 { +private: + CodeBlob* _pre_barrier_c1_runtime_code_blob; + + LIR_Opr read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + +protected: + void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val); + LIR_Opr read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators); + void enqueue_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators); + LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj); + +public: + CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; } + + virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob); +}; + +#endif // SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahLRBBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahLRBBarrierSetC1.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahLRBBarrierSetC1.cpp @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/c1/shenandoahLRBBarrierSetC1.hpp" + +#ifdef ASSERT +#define __ gen->lir(__FILE__, __LINE__)-> +#else +#define __ gen->lir()-> +#endif + +void ShenandoahLRBBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) { + if (access.is_oop()) { + if (ShenandoahSATBBarrier) { + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), access.resolved_addr(), LIR_OprFact::illegalOpr /* pre_val */); + } + enqueue_barrier(access.gen(), value, access.access_emit_info(), access.decorators()); + } + ShenandoahBaseBarrierSetC1::store_at_resolved(access, value); +} + +void ShenandoahLRBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { + if (access.is_oop()) { + LIRGenerator* gen = access.gen(); + LIR_Opr result_tmp = gen->new_register(T_OBJECT); + ShenandoahBaseBarrierSetC1::load_at_resolved(access, result_tmp); + result_tmp = write_barrier(access.gen(), result_tmp, access.access_emit_info(), + true /*access.needs_null_check()*/); + gen->lir()->move(result_tmp, result); + } else { + ShenandoahBaseBarrierSetC1::load_at_resolved(access, result); + } + if (ShenandoahKeepAliveBarrier) { + DecoratorSet decorators = access.decorators(); + bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; + bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; + LIRGenerator *gen = access.gen(); + if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { + // Register the value in the referent field with the pre-barrier + LabelObj *Lcont_anonymous; + if (is_anonymous) { + Lcont_anonymous = new LabelObj(); + generate_referent_check(access, Lcont_anonymous); + } + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr /* addr_opr */, + result /* pre_val */); + if (is_anonymous) { + __ branch_destination(Lcont_anonymous->label()); + } + } + } +} + +LIR_Opr ShenandoahLRBBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) { + if (access.is_oop()) { + if (ShenandoahSATBBarrier) { + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), access.resolved_addr(), + LIR_OprFact::illegalOpr /* pre_val */); + } + enqueue_barrier(access.gen(), new_value.result(), access.access_emit_info(), access.decorators()); + } + return ShenandoahBaseBarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); +} + +LIR_Opr ShenandoahLRBBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) { + if (access.is_oop()) { + if (ShenandoahSATBBarrier) { + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), access.resolved_addr(), + LIR_OprFact::illegalOpr /* pre_val */); + } + enqueue_barrier(access.gen(), value.result(), access.access_emit_info(), access.decorators()); + } + LIR_Opr result = ShenandoahBaseBarrierSetC1::atomic_xchg_at_resolved(access, value); + if (access.is_oop()) { + result = write_barrier(access.gen(), result, access.access_emit_info(), true); + } + return result; +} + diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahLRBBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahLRBBarrierSetC1.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahLRBBarrierSetC1.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHLRBBARRIERSETC1_HPP +#define SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHLRBBARRIERSETC1_HPP + +#include "gc/shenandoah/c1/shenandoahBaseBarrierSetC1.hpp" + +class ShenandoahLRBBarrierSetC1 : public ShenandoahBaseBarrierSetC1 { +protected: + virtual void store_at_resolved(LIRAccess& access, LIR_Opr value); + virtual void load_at_resolved(LIRAccess& access, LIR_Opr result); + virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value); + virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value); +}; + +#endif // SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHLRBBARRIERSETC1_HPP diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -32,522 +32,6 @@ #include "opto/macro.hpp" #include "opto/narrowptrnode.hpp" -ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() { - return reinterpret_cast(BarrierSet::barrier_set()->barrier_set_c2()); -} - -ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) - : _shenandoah_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { -} - -int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const { - return _shenandoah_barriers->length(); -} - -ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const { - return _shenandoah_barriers->at(idx); -} - - -void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list"); - _shenandoah_barriers->append(n); -} - -void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - if (_shenandoah_barriers->contains(n)) { - _shenandoah_barriers->remove(n); - } -} - -#define __ kit-> - -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahReadBarrier) { - obj = shenandoah_read_barrier_impl(kit, obj, false, true, true); - } - return obj; -} - -Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahStoreValEnqueueBarrier) { - obj = shenandoah_write_barrier(kit, obj); - obj = shenandoah_enqueue_barrier(kit, obj); - } - if (ShenandoahStoreValReadBarrier) { - obj = shenandoah_read_barrier_impl(kit, obj, true, false, false); - } - return obj; -} - -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_acmp(GraphKit* kit, Node* obj) { - return shenandoah_read_barrier_impl(kit, obj, true, true, false); -} - -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const { - - const Type* obj_type = obj->bottom_type(); - if (obj_type->higher_equal(TypePtr::NULL_PTR)) { - return obj; - } - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory(); - - if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) { - // We know it is null, no barrier needed. - return obj; - } - - - if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) { - - // We don't know if it's null or not. Need null-check. - enum { _not_null_path = 1, _null_path, PATH_LIMIT }; - RegionNode* region = new RegionNode(PATH_LIMIT); - Node* phi = new PhiNode(region, obj_type); - Node* null_ctrl = __ top(); - Node* not_null_obj = __ null_check_oop(obj, &null_ctrl); - - region->init_req(_null_path, null_ctrl); - phi ->init_req(_null_path, __ zerocon(T_OBJECT)); - - Node* ctrl = use_ctrl ? __ control() : NULL; - ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace); - Node* n = __ gvn().transform(rb); - - region->init_req(_not_null_path, __ control()); - phi ->init_req(_not_null_path, n); - - __ set_control(__ gvn().transform(region)); - __ record_for_igvn(region); - return __ gvn().transform(phi); - - } else { - // We know it is not null. Simple barrier is sufficient. - Node* ctrl = use_ctrl ? __ control() : NULL; - ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace); - Node* n = __ gvn().transform(rb); - __ record_for_igvn(n); - return n; - } -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const { - ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj); - Node* n = __ gvn().transform(wb); - if (n == wb) { // New barrier needs memory projection. - Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n)); - __ set_memory(proj, adr_type); - } - return n; -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const { - - if (ShenandoahWriteBarrier) { - obj = shenandoah_write_barrier_impl(kit, obj); - } - return obj; -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const { - if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) { - return obj; - } - const Type* obj_type = obj->bottom_type(); - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type); - __ record_for_igvn(n); - return n; -} - -bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr, - BasicType bt, uint adr_idx) const { - intptr_t offset = 0; - Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset); - AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase); - - if (offset == Type::OffsetBot) { - return false; // cannot unalias unless there are precise offsets - } - - if (alloc == NULL) { - return false; // No allocation found - } - - intptr_t size_in_bytes = type2aelembytes(bt); - - Node* mem = __ memory(adr_idx); // start searching here... - - for (int cnt = 0; cnt < 50; cnt++) { - - if (mem->is_Store()) { - - Node* st_adr = mem->in(MemNode::Address); - intptr_t st_offset = 0; - Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset); - - if (st_base == NULL) { - break; // inscrutable pointer - } - - // Break we have found a store with same base and offset as ours so break - if (st_base == base && st_offset == offset) { - break; - } - - if (st_offset != offset && st_offset != Type::OffsetBot) { - const int MAX_STORE = BytesPerLong; - if (st_offset >= offset + size_in_bytes || - st_offset <= offset - MAX_STORE || - st_offset <= offset - mem->as_Store()->memory_size()) { - // Success: The offsets are provably independent. - // (You may ask, why not just test st_offset != offset and be done? - // The answer is that stores of different sizes can co-exist - // in the same sequence of RawMem effects. We sometimes initialize - // a whole 'tile' of array elements with a single jint or jlong.) - mem = mem->in(MemNode::Memory); - continue; // advance through independent store memory - } - } - - if (st_base != base - && MemNode::detect_ptr_independence(base, alloc, st_base, - AllocateNode::Ideal_allocation(st_base, phase), - phase)) { - // Success: The bases are provably independent. - mem = mem->in(MemNode::Memory); - continue; // advance through independent store memory - } - } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) { - - InitializeNode* st_init = mem->in(0)->as_Initialize(); - AllocateNode* st_alloc = st_init->allocation(); - - // Make sure that we are looking at the same allocation site. - // The alloc variable is guaranteed to not be null here from earlier check. - if (alloc == st_alloc) { - // Check that the initialization is storing NULL so that no previous store - // has been moved up and directly write a reference - Node* captured_store = st_init->find_captured_store(offset, - type2aelembytes(T_OBJECT), - phase); - if (captured_store == NULL || captured_store == st_init->zero_memory()) { - return true; - } - } - } - - // Unless there is an explicit 'continue', we must bail out here, - // because 'mem' is an inscrutable memory state (e.g., a call). - break; - } - - return false; -} - -#undef __ -#define __ ideal. - -void ShenandoahBarrierSetC2::satb_write_barrier_pre(GraphKit* kit, - bool do_load, - Node* obj, - Node* adr, - uint alias_idx, - Node* val, - const TypeOopPtr* val_type, - Node* pre_val, - BasicType bt) const { - - // Some sanity checks - // Note: val is unused in this routine. - - if (do_load) { - // We need to generate the load of the previous value - assert(obj != NULL, "must have a base"); - assert(adr != NULL, "where are loading from?"); - assert(pre_val == NULL, "loaded already?"); - assert(val_type != NULL, "need a type"); - - if (ReduceInitialCardMarks - && satb_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) { - return; - } - - } else { - // In this case both val_type and alias_idx are unused. - assert(pre_val != NULL, "must be loaded already"); - // Nothing to be done if pre_val is null. - if (pre_val->bottom_type() == TypePtr::NULL_PTR) return; - assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here"); - } - assert(bt == T_OBJECT, "or we shouldn't be here"); - - IdealKit ideal(kit, true); - - Node* tls = __ thread(); // ThreadLocalStorage - - Node* no_base = __ top(); - Node* zero = __ ConI(0); - Node* zeroX = __ ConX(0); - - float likely = PROB_LIKELY(0.999); - float unlikely = PROB_UNLIKELY(0.999); - - // Offsets into the thread - const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()); - const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()); - - // Now the actual pointers into the thread - Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset)); - Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset)); - - // Now some of the values - Node* marking; - Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()))); - Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); - marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING)); - assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape"); - - // if (!marking) - __ if_then(marking, BoolTest::ne, zero, unlikely); { - BasicType index_bt = TypeX_X->basic_type(); - assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); - Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw); - - if (do_load) { - // load original value - // alias_idx correct?? - pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx); - } - - // if (pre_val != NULL) - __ if_then(pre_val, BoolTest::ne, kit->null()); { - Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw); - - // is the queue for this thread full? - __ if_then(index, BoolTest::ne, zeroX, likely); { - - // decrement the index - Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t)))); - - // Now get the buffer location we will log the previous value into and store it - Node *log_addr = __ AddP(no_base, buffer, next_index); - __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered); - // update the index - __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered); - - } __ else_(); { - - // logging buffer is full, call the runtime - const TypeFunc *tf = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(); - __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", pre_val, tls); - } __ end_if(); // (!index) - } __ end_if(); // (pre_val != NULL) - } __ end_if(); // (!marking) - - // Final sync IdealKit and GraphKit. - kit->final_sync(ideal); - - if (ShenandoahSATBBarrier && adr != NULL) { - Node* c = kit->control(); - Node* call = c->in(1)->in(1)->in(1)->in(0); - assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected"); - call->add_req(adr); - } -} - -bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) { - return call->is_CallLeaf() && - call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry); -} - -bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) { - - if (n->Opcode() != Op_If) { - return false; - } - - Node* bol = n->in(1); - assert(bol->is_Bool(), ""); - Node* cmpx = bol->in(1); - if (bol->as_Bool()->_test._test == BoolTest::ne && - cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) && - is_shenandoah_state_load(cmpx->in(1)->in(1)) && - cmpx->in(1)->in(2)->is_Con() && - cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) { - return true; - } - - return false; -} - -bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) { - if (!n->is_Load()) return false; - const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset()); - return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal - && n->in(2)->in(3)->is_Con() - && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset; -} - -void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit, - bool do_load, - Node* obj, - Node* adr, - uint alias_idx, - Node* val, - const TypeOopPtr* val_type, - Node* pre_val, - BasicType bt) const { - if (ShenandoahSATBBarrier) { - IdealKit ideal(kit); - kit->sync_kit(ideal); - - satb_write_barrier_pre(kit, do_load, obj, adr, alias_idx, val, val_type, pre_val, bt); - - ideal.sync_kit(kit); - kit->final_sync(ideal); - } -} - -Node* ShenandoahBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const { - return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val)); -} - -// Helper that guards and inserts a pre-barrier. -void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset, - Node* pre_val, bool need_mem_bar) const { - // We could be accessing the referent field of a reference object. If so, when G1 - // is enabled, we need to log the value in the referent field in an SATB buffer. - // This routine performs some compile time filters and generates suitable - // runtime filters that guard the pre-barrier code. - // Also add memory barrier for non volatile load from the referent field - // to prevent commoning of loads across safepoint. - - // Some compile time checks. - - // If offset is a constant, is it java_lang_ref_Reference::_reference_offset? - const TypeX* otype = offset->find_intptr_t_type(); - if (otype != NULL && otype->is_con() && - otype->get_con() != java_lang_ref_Reference::referent_offset) { - // Constant offset but not the reference_offset so just return - return; - } - - // We only need to generate the runtime guards for instances. - const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr(); - if (btype != NULL) { - if (btype->isa_aryptr()) { - // Array type so nothing to do - return; - } - - const TypeInstPtr* itype = btype->isa_instptr(); - if (itype != NULL) { - // Can the klass of base_oop be statically determined to be - // _not_ a sub-class of Reference and _not_ Object? - ciKlass* klass = itype->klass(); - if ( klass->is_loaded() && - !klass->is_subtype_of(kit->env()->Reference_klass()) && - !kit->env()->Object_klass()->is_subtype_of(klass)) { - return; - } - } - } - - // The compile time filters did not reject base_oop/offset so - // we need to generate the following runtime filters - // - // if (offset == java_lang_ref_Reference::_reference_offset) { - // if (instance_of(base, java.lang.ref.Reference)) { - // pre_barrier(_, pre_val, ...); - // } - // } - - float likely = PROB_LIKELY( 0.999); - float unlikely = PROB_UNLIKELY(0.999); - - IdealKit ideal(kit); - - Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset); - - __ if_then(offset, BoolTest::eq, referent_off, unlikely); { - // Update graphKit memory and control from IdealKit. - kit->sync_kit(ideal); - - Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass())); - Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con); - - // Update IdealKit memory and control from graphKit. - __ sync_kit(kit); - - Node* one = __ ConI(1); - // is_instof == 0 if base_oop == NULL - __ if_then(is_instof, BoolTest::eq, one, unlikely); { - - // Update graphKit from IdeakKit. - kit->sync_kit(ideal); - - // Use the pre-barrier to record the value in the referent field - satb_write_barrier_pre(kit, false /* do_load */, - NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */, - pre_val /* pre_val */, - T_OBJECT); - if (need_mem_bar) { - // Add memory barrier to prevent commoning reads from this field - // across safepoint since GC can change its value. - kit->insert_mem_bar(Op_MemBarCPUOrder); - } - // Update IdealKit from graphKit. - __ sync_kit(kit); - - } __ end_if(); // _ref_type != ref_none - } __ end_if(); // offset == referent_offset - - // Final sync IdealKit and GraphKit. - kit->final_sync(ideal); -} - -#undef __ - -const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() { - const Type **fields = TypeTuple::fields(2); - fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value - fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); - - // create result type (range) - fields = TypeTuple::fields(0); - const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); - - return TypeFunc::make(domain, range); -} - -const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() { - const Type **fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); - - // create result type (range) - fields = TypeTuple::fields(0); - const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); - - return TypeFunc::make(domain, range); -} - -const TypeFunc* ShenandoahBarrierSetC2::shenandoah_write_barrier_Type() { - const Type **fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); - - // create result type (range) - fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; - const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); - - return TypeFunc::make(domain, range); -} - void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const { const TypePtr* adr_type = access.addr().type(); @@ -773,12 +257,6 @@ return result; } -void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { - assert(!src->is_AddP(), "unexpected input"); - src = shenandoah_read_barrier(kit, src); - BarrierSetC2::clone(kit, src, dst, size, is_array); -} - Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const { bool is_write = decorators & ACCESS_WRITE; if (is_write) { @@ -799,146 +277,3 @@ } } } - -// Support for GC barriers emitted during parsing -bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const { - if (node->Opcode() != Op_CallLeaf) { - return false; - } - CallLeafNode *call = node->as_CallLeaf(); - if (call->_name == NULL) { - return false; - } - - return strcmp(call->_name, "shenandoah_clone_barrier") == 0 || - strcmp(call->_name, "shenandoah_cas_obj") == 0 || - strcmp(call->_name, "shenandoah_wb_pre") == 0; -} - -Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const { - // Currently not needed. - return c; -} - -Node* ShenandoahBarrierSetC2::peek_thru_gc_barrier(Node* v) const { - return ShenandoahBarrierNode::skip_through_barrier(v); -} - -bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { - bool is_oop = type == T_OBJECT || type == T_ARRAY; - if (!is_oop) { - return false; - } - - if (tightly_coupled_alloc) { - if (phase == Optimization) { - return false; - } - return !is_clone; - } - if (phase == Optimization) { - return !ShenandoahStoreValEnqueueBarrier; - } - return true; -} - -// Support for macro expanded GC barriers -void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); - } -} - -void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); - } -} - -void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const { - if (is_shenandoah_wb_pre_call(n)) { - shenandoah_eliminate_wb_pre(n, ¯o->igvn()); - } -} - -void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const { - assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), ""); - Node* c = call->as_Call()->proj_out(TypeFunc::Control); - c = c->unique_ctrl_out(); - assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?"); - c = c->unique_ctrl_out(); - assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?"); - Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0); - assert(iff->is_If(), "expect test"); - if (!is_shenandoah_marking_if(igvn, iff)) { - c = c->unique_ctrl_out(); - assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?"); - iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0); - assert(is_shenandoah_marking_if(igvn, iff), "expect marking test"); - } - Node* cmpx = iff->in(1)->in(1); - igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ)); - igvn->rehash_node_delayed(call); - call->del_req(call->req()-1); -} - -void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { -} - -void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { - for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) { - ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i); - if (!useful.member(n)) { - state()->remove_shenandoah_barrier(n); - } - } - -} - -void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {} - -void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const { - return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena); -} - -ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const { - return reinterpret_cast(Compile::current()->barrier_set_state()); -} - -// If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be -// expanded later, then now is the time to do so. -bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; } -void ShenandoahBarrierSetC2::verify_gc_barriers(bool post_parse) const { -#ifdef ASSERT - if (ShenandoahVerifyOptoBarriers && !post_parse) { - ShenandoahBarrierNode::verify(Compile::current()->root()); - } -#endif -} - -Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN *phase, Node* n, bool can_reshape) const { - if (is_shenandoah_wb_pre_call(n)) { - uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt(); - if (n->req() > cnt) { - Node* addp = n->in(cnt); - if (has_only_shenandoah_wb_pre_uses(addp)) { - n->del_req(cnt); - if (can_reshape) { - phase->is_IterGVN()->_worklist.push(addp); - } - return n; - } - } - } - return NULL; -} - -bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) { - for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { - Node* u = n->fast_out(i); - if (!is_shenandoah_wb_pre_call(u)) { - return false; - } - } - return n->outcnt() > 0; -} diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -24,57 +24,9 @@ #ifndef SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP #define SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP -#include "gc/shared/c2/barrierSetC2.hpp" +#include "gc/shenandoah/c2/shenandoahBaseBarrierSetC2.hpp" -class ShenandoahBarrierSetC2State : public ResourceObj { -private: - GrowableArray* _shenandoah_barriers; - - public: - ShenandoahBarrierSetC2State(Arena* comp_arena); - int shenandoah_barriers_count() const; - ShenandoahWriteBarrierNode* shenandoah_barrier(int idx) const; - void add_shenandoah_barrier(ShenandoahWriteBarrierNode * n); - void remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n); -}; - -class ShenandoahBarrierSetC2 : public BarrierSetC2 { -private: - - void shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const; - - bool satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr, - BasicType bt, uint adr_idx) const; - void satb_write_barrier_pre(GraphKit* kit, bool do_load, - Node* obj, - Node* adr, - uint alias_idx, - Node* val, - const TypeOopPtr* val_type, - Node* pre_val, - BasicType bt) const; - - void shenandoah_write_barrier_pre(GraphKit* kit, - bool do_load, - Node* obj, - Node* adr, - uint alias_idx, - Node* val, - const TypeOopPtr* val_type, - Node* pre_val, - BasicType bt) const; - - Node* shenandoah_enqueue_barrier(GraphKit* kit, Node* val) const; - Node* shenandoah_read_barrier(GraphKit* kit, Node* obj) const; - Node* shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const; - Node* shenandoah_write_barrier(GraphKit* kit, Node* obj) const; - Node* shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const; - Node* shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const; - Node* shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const; - - void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset, - Node* pre_val, bool need_mem_bar) const; - +class ShenandoahBarrierSetC2 : public ShenandoahBaseBarrierSetC2 { protected: virtual void resolve_address(C2Access& access) const; virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; @@ -86,54 +38,10 @@ virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const; public: - static ShenandoahBarrierSetC2* bsc2(); - - static bool is_shenandoah_wb_pre_call(Node* call); - static bool is_shenandoah_marking_if(PhaseTransform *phase, Node* n); - static bool is_shenandoah_state_load(Node* n); - static bool has_only_shenandoah_wb_pre_uses(Node* n); - - ShenandoahBarrierSetC2State* state() const; - - Node* shenandoah_read_barrier_acmp(GraphKit* kit, Node* obj); - - - static const TypeFunc* write_ref_field_pre_entry_Type(); - static const TypeFunc* shenandoah_clone_barrier_Type(); - static const TypeFunc* shenandoah_write_barrier_Type(); - - // This is the entry-point for the backend to perform accesses through the Access API. - virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const; virtual Node* resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const; virtual void resolve_for_obj_equals(GraphKit* kit, Node*& a, Node*& b) const; - - // These are general helper methods used by C2 - virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const; - - // Support for GC barriers emitted during parsing - virtual bool is_gc_barrier_node(Node* node) const; - virtual Node* step_over_gc_barrier(Node* c) const; - virtual Node* peek_thru_gc_barrier(Node* v) const; - - // Support for macro expanded GC barriers - virtual void register_potential_barrier_node(Node* node) const; - virtual void unregister_potential_barrier_node(Node* node) const; - virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const; - virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const; - virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const; - virtual void add_users_to_worklist(Unique_Node_List* worklist) const; - - // Allow barrier sets to have shared state that is preserved across a compilation unit. - // This could for example comprise macro nodes to be expanded during macro expansion. - virtual void* create_barrier_state(Arena* comp_arena) const; - // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be - // expanded later, then now is the time to do so. - virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const; - virtual void verify_gc_barriers(bool post_parse) const; - - virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const; }; #endif // SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBaseBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBaseBarrierSetC2.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBaseBarrierSetC2.cpp @@ -0,0 +1,704 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeuristics.hpp" +#include "gc/shenandoah/shenandoahRuntime.hpp" +#include "gc/shenandoah/c2/shenandoahBaseBarrierSetC2.hpp" +#include "gc/shenandoah/c2/shenandoahSupport.hpp" +#include "opto/graphKit.hpp" +#include "opto/idealKit.hpp" +#include "opto/macro.hpp" +#include "opto/narrowptrnode.hpp" + +ShenandoahBaseBarrierSetC2* ShenandoahBaseBarrierSetC2::bsc2() { + return reinterpret_cast(BarrierSet::barrier_set()->barrier_set_c2()); +} + +ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) + : _shenandoah_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { +} + +int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const { + return _shenandoah_barriers->length(); +} + +ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const { + return _shenandoah_barriers->at(idx); +} + + +void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { + assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list"); + _shenandoah_barriers->append(n); +} + +void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { + if (_shenandoah_barriers->contains(n)) { + _shenandoah_barriers->remove(n); + } +} + +#define __ kit-> + +Node* ShenandoahBaseBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const { + if (ShenandoahReadBarrier) { + obj = shenandoah_read_barrier_impl(kit, obj, false, true, true); + } + return obj; +} + +Node* ShenandoahBaseBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const { + if (ShenandoahStoreValEnqueueBarrier) { + obj = shenandoah_write_barrier(kit, obj); + obj = shenandoah_enqueue_barrier(kit, obj); + } + if (ShenandoahStoreValReadBarrier) { + obj = shenandoah_read_barrier_impl(kit, obj, true, false, false); + } + return obj; +} + +Node* ShenandoahBaseBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const { + + const Type* obj_type = obj->bottom_type(); + if (obj_type->higher_equal(TypePtr::NULL_PTR)) { + return obj; + } + const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); + Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory(); + + if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) { + // We know it is null, no barrier needed. + return obj; + } + + + if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) { + + // We don't know if it's null or not. Need null-check. + enum { _not_null_path = 1, _null_path, PATH_LIMIT }; + RegionNode* region = new RegionNode(PATH_LIMIT); + Node* phi = new PhiNode(region, obj_type); + Node* null_ctrl = __ top(); + Node* not_null_obj = __ null_check_oop(obj, &null_ctrl); + + region->init_req(_null_path, null_ctrl); + phi ->init_req(_null_path, __ zerocon(T_OBJECT)); + + Node* ctrl = use_ctrl ? __ control() : NULL; + ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace); + Node* n = __ gvn().transform(rb); + + region->init_req(_not_null_path, __ control()); + phi ->init_req(_not_null_path, n); + + __ set_control(__ gvn().transform(region)); + __ record_for_igvn(region); + return __ gvn().transform(phi); + + } else { + // We know it is not null. Simple barrier is sufficient. + Node* ctrl = use_ctrl ? __ control() : NULL; + ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace); + Node* n = __ gvn().transform(rb); + __ record_for_igvn(n); + return n; + } +} + +Node* ShenandoahBaseBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const { + ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj); + Node* n = __ gvn().transform(wb); + if (n == wb) { // New barrier needs memory projection. + Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n)); + __ set_memory(proj, adr_type); + } + return n; +} + +Node* ShenandoahBaseBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const { + + if (ShenandoahWriteBarrier) { + obj = shenandoah_write_barrier_impl(kit, obj); + } + return obj; +} + +Node* ShenandoahBaseBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const { + if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) { + return obj; + } + const Type* obj_type = obj->bottom_type(); + const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); + Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type); + __ record_for_igvn(n); + return n; +} + +bool ShenandoahBaseBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr, + BasicType bt, uint adr_idx) const { + intptr_t offset = 0; + Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset); + AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase); + + if (offset == Type::OffsetBot) { + return false; // cannot unalias unless there are precise offsets + } + + if (alloc == NULL) { + return false; // No allocation found + } + + intptr_t size_in_bytes = type2aelembytes(bt); + + Node* mem = __ memory(adr_idx); // start searching here... + + for (int cnt = 0; cnt < 50; cnt++) { + + if (mem->is_Store()) { + + Node* st_adr = mem->in(MemNode::Address); + intptr_t st_offset = 0; + Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset); + + if (st_base == NULL) { + break; // inscrutable pointer + } + + // Break we have found a store with same base and offset as ours so break + if (st_base == base && st_offset == offset) { + break; + } + + if (st_offset != offset && st_offset != Type::OffsetBot) { + const int MAX_STORE = BytesPerLong; + if (st_offset >= offset + size_in_bytes || + st_offset <= offset - MAX_STORE || + st_offset <= offset - mem->as_Store()->memory_size()) { + // Success: The offsets are provably independent. + // (You may ask, why not just test st_offset != offset and be done? + // The answer is that stores of different sizes can co-exist + // in the same sequence of RawMem effects. We sometimes initialize + // a whole 'tile' of array elements with a single jint or jlong.) + mem = mem->in(MemNode::Memory); + continue; // advance through independent store memory + } + } + + if (st_base != base + && MemNode::detect_ptr_independence(base, alloc, st_base, + AllocateNode::Ideal_allocation(st_base, phase), + phase)) { + // Success: The bases are provably independent. + mem = mem->in(MemNode::Memory); + continue; // advance through independent store memory + } + } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) { + + InitializeNode* st_init = mem->in(0)->as_Initialize(); + AllocateNode* st_alloc = st_init->allocation(); + + // Make sure that we are looking at the same allocation site. + // The alloc variable is guaranteed to not be null here from earlier check. + if (alloc == st_alloc) { + // Check that the initialization is storing NULL so that no previous store + // has been moved up and directly write a reference + Node* captured_store = st_init->find_captured_store(offset, + type2aelembytes(T_OBJECT), + phase); + if (captured_store == NULL || captured_store == st_init->zero_memory()) { + return true; + } + } + } + + // Unless there is an explicit 'continue', we must bail out here, + // because 'mem' is an inscrutable memory state (e.g., a call). + break; + } + + return false; +} + +#undef __ +#define __ ideal. + +void ShenandoahBaseBarrierSetC2::satb_write_barrier_pre(GraphKit* kit, + bool do_load, + Node* obj, + Node* adr, + uint alias_idx, + Node* val, + const TypeOopPtr* val_type, + Node* pre_val, + BasicType bt) const { + + // Some sanity checks + // Note: val is unused in this routine. + + if (do_load) { + // We need to generate the load of the previous value + assert(obj != NULL, "must have a base"); + assert(adr != NULL, "where are loading from?"); + assert(pre_val == NULL, "loaded already?"); + assert(val_type != NULL, "need a type"); + + if (ReduceInitialCardMarks + && satb_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) { + return; + } + + } else { + // In this case both val_type and alias_idx are unused. + assert(pre_val != NULL, "must be loaded already"); + // Nothing to be done if pre_val is null. + if (pre_val->bottom_type() == TypePtr::NULL_PTR) return; + assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here"); + } + assert(bt == T_OBJECT, "or we shouldn't be here"); + + IdealKit ideal(kit, true); + + Node* tls = __ thread(); // ThreadLocalStorage + + Node* no_ctrl = NULL; + Node* no_base = __ top(); + Node* zero = __ ConI(0); + Node* zeroX = __ ConX(0); + + float likely = PROB_LIKELY(0.999); + float unlikely = PROB_UNLIKELY(0.999); + + BasicType active_type = in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE; + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 || in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "flag width"); + + // Offsets into the thread + const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()); + const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()); + const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()); + + // Now the actual pointers into the thread + Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset)); + Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset)); + Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset)); + + // Now some of the values + Node* marking; + Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()))); + Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); + marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING)); + assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape"); + + // if (!marking) + __ if_then(marking, BoolTest::ne, zero, unlikely); { + BasicType index_bt = TypeX_X->basic_type(); + assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); + Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw); + + if (do_load) { + // load original value + // alias_idx correct?? + pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx); + } + + // if (pre_val != NULL) + __ if_then(pre_val, BoolTest::ne, kit->null()); { + Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw); + + // is the queue for this thread full? + __ if_then(index, BoolTest::ne, zeroX, likely); { + + // decrement the index + Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t)))); + + // Now get the buffer location we will log the previous value into and store it + Node *log_addr = __ AddP(no_base, buffer, next_index); + __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered); + // update the index + __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered); + + } __ else_(); { + + // logging buffer is full, call the runtime + const TypeFunc *tf = ShenandoahBaseBarrierSetC2::write_ref_field_pre_entry_Type(); + __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", pre_val, tls); + } __ end_if(); // (!index) + } __ end_if(); // (pre_val != NULL) + } __ end_if(); // (!marking) + + // Final sync IdealKit and GraphKit. + kit->final_sync(ideal); + + if (ShenandoahSATBBarrier && adr != NULL) { + Node* c = kit->control(); + Node* call = c->in(1)->in(1)->in(1)->in(0); + assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected"); + call->add_req(adr); + } +} + +bool ShenandoahBaseBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) { + return call->is_CallLeaf() && + call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry); +} + +bool ShenandoahBaseBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) { + + if (n->Opcode() != Op_If) { + return false; + } + + Node* bol = n->in(1); + assert(bol->is_Bool(), ""); + Node* cmpx = bol->in(1); + if (bol->as_Bool()->_test._test == BoolTest::ne && + cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) && + is_shenandoah_state_load(cmpx->in(1)->in(1)) && + cmpx->in(1)->in(2)->is_Con() && + cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) { + return true; + } + + return false; +} + +bool ShenandoahBaseBarrierSetC2::is_shenandoah_state_load(Node* n) { + if (!n->is_Load()) return false; + const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset()); + return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal + && n->in(2)->in(3)->is_Con() + && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset; +} + +void ShenandoahBaseBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit, + bool do_load, + Node* obj, + Node* adr, + uint alias_idx, + Node* val, + const TypeOopPtr* val_type, + Node* pre_val, + BasicType bt) const { + if (ShenandoahSATBBarrier) { + IdealKit ideal(kit); + kit->sync_kit(ideal); + + satb_write_barrier_pre(kit, do_load, obj, adr, alias_idx, val, val_type, pre_val, bt); + + ideal.sync_kit(kit); + kit->final_sync(ideal); + } +} + +Node* ShenandoahBaseBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const { + if (ShenandoahStoreValEnqueueBarrier) { + return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val)); + } else { + return pre_val; + } +} + +// Helper that guards and inserts a pre-barrier. +void ShenandoahBaseBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset, + Node* pre_val, bool need_mem_bar) const { + // We could be accessing the referent field of a reference object. If so, when G1 + // is enabled, we need to log the value in the referent field in an SATB buffer. + // This routine performs some compile time filters and generates suitable + // runtime filters that guard the pre-barrier code. + // Also add memory barrier for non volatile load from the referent field + // to prevent commoning of loads across safepoint. + + // Some compile time checks. + + // If offset is a constant, is it java_lang_ref_Reference::_reference_offset? + const TypeX* otype = offset->find_intptr_t_type(); + if (otype != NULL && otype->is_con() && + otype->get_con() != java_lang_ref_Reference::referent_offset) { + // Constant offset but not the reference_offset so just return + return; + } + + // We only need to generate the runtime guards for instances. + const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr(); + if (btype != NULL) { + if (btype->isa_aryptr()) { + // Array type so nothing to do + return; + } + + const TypeInstPtr* itype = btype->isa_instptr(); + if (itype != NULL) { + // Can the klass of base_oop be statically determined to be + // _not_ a sub-class of Reference and _not_ Object? + ciKlass* klass = itype->klass(); + if ( klass->is_loaded() && + !klass->is_subtype_of(kit->env()->Reference_klass()) && + !kit->env()->Object_klass()->is_subtype_of(klass)) { + return; + } + } + } + + // The compile time filters did not reject base_oop/offset so + // we need to generate the following runtime filters + // + // if (offset == java_lang_ref_Reference::_reference_offset) { + // if (instance_of(base, java.lang.ref.Reference)) { + // pre_barrier(_, pre_val, ...); + // } + // } + + float likely = PROB_LIKELY( 0.999); + float unlikely = PROB_UNLIKELY(0.999); + + IdealKit ideal(kit); + + Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset); + + __ if_then(offset, BoolTest::eq, referent_off, unlikely); { + // Update graphKit memory and control from IdealKit. + kit->sync_kit(ideal); + + Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass())); + Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con); + + // Update IdealKit memory and control from graphKit. + __ sync_kit(kit); + + Node* one = __ ConI(1); + // is_instof == 0 if base_oop == NULL + __ if_then(is_instof, BoolTest::eq, one, unlikely); { + + // Update graphKit from IdeakKit. + kit->sync_kit(ideal); + + // Use the pre-barrier to record the value in the referent field + satb_write_barrier_pre(kit, false /* do_load */, + NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */, + pre_val /* pre_val */, + T_OBJECT); + if (need_mem_bar) { + // Add memory barrier to prevent commoning reads from this field + // across safepoint since GC can change its value. + kit->insert_mem_bar(Op_MemBarCPUOrder); + } + // Update IdealKit from graphKit. + __ sync_kit(kit); + + } __ end_if(); // _ref_type != ref_none + } __ end_if(); // offset == referent_offset + + // Final sync IdealKit and GraphKit. + kit->final_sync(ideal); +} + +#undef __ + +const TypeFunc* ShenandoahBaseBarrierSetC2::write_ref_field_pre_entry_Type() { + const Type **fields = TypeTuple::fields(2); + fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value + fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); + + // create result type (range) + fields = TypeTuple::fields(0); + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); + + return TypeFunc::make(domain, range); +} + +const TypeFunc* ShenandoahBaseBarrierSetC2::shenandoah_clone_barrier_Type() { + const Type **fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); + + // create result type (range) + fields = TypeTuple::fields(0); + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); + + return TypeFunc::make(domain, range); +} + +const TypeFunc* ShenandoahBaseBarrierSetC2::shenandoah_write_barrier_Type() { + const Type **fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); + + // create result type (range) + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); + + return TypeFunc::make(domain, range); +} + +void ShenandoahBaseBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { + assert(!src->is_AddP(), "unexpected input"); + src = shenandoah_read_barrier(kit, src); + BarrierSetC2::clone(kit, src, dst, size, is_array); +} + +// Support for GC barriers emitted during parsing +bool ShenandoahBaseBarrierSetC2::is_gc_barrier_node(Node* node) const { + if (node->Opcode() != Op_CallLeaf) { + return false; + } + CallLeafNode *call = node->as_CallLeaf(); + if (call->_name == NULL) { + return false; + } + + return strcmp(call->_name, "shenandoah_clone_barrier") == 0 || + strcmp(call->_name, "shenandoah_cas_obj") == 0 || + strcmp(call->_name, "shenandoah_wb_pre") == 0; +} + +Node* ShenandoahBaseBarrierSetC2::step_over_gc_barrier(Node* c) const { + // Currently not needed. + return c; +} + +Node* ShenandoahBaseBarrierSetC2::peek_thru_gc_barrier(Node* v) const { + return ShenandoahBarrierNode::skip_through_barrier(v); +} + +bool ShenandoahBaseBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { + bool is_oop = type == T_OBJECT || type == T_ARRAY; + if (!is_oop) { + return false; + } + + if (tightly_coupled_alloc) { + if (phase == Optimization) { + return false; + } + return !is_clone; + } + if (phase == Optimization) { + return !ShenandoahStoreValEnqueueBarrier; + } + return true; +} + +// Support for macro expanded GC barriers +void ShenandoahBaseBarrierSetC2::register_potential_barrier_node(Node* node) const { + if (node->Opcode() == Op_ShenandoahWriteBarrier) { + state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + } +} + +void ShenandoahBaseBarrierSetC2::unregister_potential_barrier_node(Node* node) const { + if (node->Opcode() == Op_ShenandoahWriteBarrier) { + state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + } +} + +void ShenandoahBaseBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const { + if (is_shenandoah_wb_pre_call(n)) { + shenandoah_eliminate_wb_pre(n, ¯o->igvn()); + } +} + +void ShenandoahBaseBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const { + assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), ""); + Node* c = call->as_Call()->proj_out(TypeFunc::Control); + c = c->unique_ctrl_out(); + assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?"); + c = c->unique_ctrl_out(); + assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?"); + Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0); + assert(iff->is_If(), "expect test"); + if (!is_shenandoah_marking_if(igvn, iff)) { + c = c->unique_ctrl_out(); + assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?"); + iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0); + assert(is_shenandoah_marking_if(igvn, iff), "expect marking test"); + } + Node* cmpx = iff->in(1)->in(1); + igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ)); + igvn->rehash_node_delayed(call); + call->del_req(call->req()-1); +} + +void ShenandoahBaseBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { +} + +void ShenandoahBaseBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { + for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) { + ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i); + if (!useful.member(n)) { + state()->remove_shenandoah_barrier(n); + } + } + +} + +void ShenandoahBaseBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {} + +void* ShenandoahBaseBarrierSetC2::create_barrier_state(Arena* comp_arena) const { + return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena); +} + +ShenandoahBarrierSetC2State* ShenandoahBaseBarrierSetC2::state() const { + return reinterpret_cast(Compile::current()->barrier_set_state()); +} + +// If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be +// expanded later, then now is the time to do so. +bool ShenandoahBaseBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; } +void ShenandoahBaseBarrierSetC2::verify_gc_barriers(bool post_parse) const { +#ifdef ASSERT + if (ShenandoahVerifyOptoBarriers && !post_parse) { + ShenandoahBarrierNode::verify(Compile::current()->root()); + } +#endif +} + +Node* ShenandoahBaseBarrierSetC2::ideal_node(PhaseGVN *phase, Node* n, bool can_reshape) const { + if (is_shenandoah_wb_pre_call(n)) { + uint cnt = ShenandoahBaseBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt(); + if (n->req() > cnt) { + Node* addp = n->in(cnt); + if (has_only_shenandoah_wb_pre_uses(addp)) { + n->del_req(cnt); + if (can_reshape) { + phase->is_IterGVN()->_worklist.push(addp); + } + return n; + } + } + } + return NULL; +} + +bool ShenandoahBaseBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) { + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { + Node* u = n->fast_out(i); + if (!is_shenandoah_wb_pre_call(u)) { + return false; + } + } + return n->outcnt() > 0; +} diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBaseBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBaseBarrierSetC2.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBaseBarrierSetC2.hpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBASEBARRIERSETC2_HPP +#define SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBASEBARRIERSETC2_HPP + +#include "gc/shared/c2/barrierSetC2.hpp" + +class ShenandoahBarrierSetC2State : public ResourceObj { +private: + GrowableArray* _shenandoah_barriers; + +public: + ShenandoahBarrierSetC2State(Arena* comp_arena); + int shenandoah_barriers_count() const; + ShenandoahWriteBarrierNode* shenandoah_barrier(int idx) const; + void add_shenandoah_barrier(ShenandoahWriteBarrierNode * n); + void remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n); +}; + +class ShenandoahBaseBarrierSetC2 : public BarrierSetC2 { +private: + + void shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const; + + bool satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr, + BasicType bt, uint adr_idx) const; + + Node* shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const; + Node* shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const; + Node* shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const; + +protected: + Node* shenandoah_read_barrier(GraphKit* kit, Node* obj) const; + Node* shenandoah_write_barrier(GraphKit* kit, Node* obj) const; + Node* shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const; + void shenandoah_write_barrier_pre(GraphKit* kit, + bool do_load, + Node* obj, + Node* adr, + uint alias_idx, + Node* val, + const TypeOopPtr* val_type, + Node* pre_val, + BasicType bt) const; + + void satb_write_barrier_pre(GraphKit* kit, bool do_load, + Node* obj, + Node* adr, + uint alias_idx, + Node* val, + const TypeOopPtr* val_type, + Node* pre_val, + BasicType bt) const; + + void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset, + Node* pre_val, bool need_mem_bar) const; + + Node* shenandoah_enqueue_barrier(GraphKit* kit, Node* val) const; + +public: + static ShenandoahBaseBarrierSetC2* bsc2(); + + static bool is_shenandoah_wb_pre_call(Node* call); + static bool is_shenandoah_marking_if(PhaseTransform *phase, Node* n); + static bool is_shenandoah_state_load(Node* n); + static bool has_only_shenandoah_wb_pre_uses(Node* n); + + ShenandoahBarrierSetC2State* state() const; + + static const TypeFunc* write_ref_field_pre_entry_Type(); + static const TypeFunc* shenandoah_clone_barrier_Type(); + static const TypeFunc* shenandoah_write_barrier_Type(); + + // This is the entry-point for the backend to perform accesses through the Access API. + virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const; + + // These are general helper methods used by C2 + virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const; + + // Support for GC barriers emitted during parsing + virtual bool is_gc_barrier_node(Node* node) const; + virtual Node* step_over_gc_barrier(Node* c) const; + virtual Node* peek_thru_gc_barrier(Node* v) const; + + // Support for macro expanded GC barriers + virtual void register_potential_barrier_node(Node* node) const; + virtual void unregister_potential_barrier_node(Node* node) const; + virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const; + virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const; + virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const; + virtual void add_users_to_worklist(Unique_Node_List* worklist) const; + + // Allow barrier sets to have shared state that is preserved across a compilation unit. + // This could for example comprise macro nodes to be expanded during macro expansion. + virtual void* create_barrier_state(Arena* comp_arena) const; + // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be + // expanded later, then now is the time to do so. + virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const; + virtual void verify_gc_barriers(bool post_parse) const; + + virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const; +}; + +#endif // SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBASEBARRIERSETC2_HPP diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahLRBBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahLRBBarrierSetC2.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahLRBBarrierSetC2.cpp @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/c2/shenandoahLRBBarrierSetC2.hpp" +#include "gc/shenandoah/c2/shenandoahSupport.hpp" +#include "opto/graphKit.hpp" +#include "opto/narrowptrnode.hpp" + +Node* ShenandoahLRBBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { + DecoratorSet decorators = access.decorators(); + GraphKit* kit = access.kit(); + + const TypePtr* adr_type = access.addr().type(); + Node* adr = access.addr().node(); + + bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; + bool on_heap = (decorators & IN_HEAP) != 0; + + if (!access.is_oop() || (!on_heap && !anonymous)) { + return ShenandoahBaseBarrierSetC2::store_at_resolved(access, val); + } + + uint adr_idx = kit->C->get_alias_index(adr_type); + assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); + val.set_node(shenandoah_enqueue_barrier(kit, val.node())); + shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(), + static_cast(val.type()), NULL /* pre_val */, access.type()); + return ShenandoahBaseBarrierSetC2::store_at_resolved(access, val); +} + +Node* ShenandoahLRBBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { + DecoratorSet decorators = access.decorators(); + GraphKit* kit = access.kit(); + + Node* adr = access.addr().node(); + Node* obj = access.base(); + + bool mismatched = (decorators & C2_MISMATCHED) != 0; + bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0; + bool on_heap = (decorators & IN_HEAP) != 0; + bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool is_unordered = (decorators & MO_UNORDERED) != 0; + bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap; + + Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : kit->top(); + Node* load = ShenandoahBaseBarrierSetC2::load_at_resolved(access, val_type); + + if (access.is_oop()) { + load = shenandoah_write_barrier(access.kit(), load); + } + + // If we are reading the value of the referent field of a Reference + // object (either by using Unsafe directly or through reflection) + // then, if SATB is enabled, we need to record the referent in an + // SATB log buffer using the pre-barrier mechanism. + // Also we need to add memory barrier to prevent commoning reads + // from this field across safepoint since GC can change its value. + bool need_read_barrier = ShenandoahKeepAliveBarrier && + (on_heap && (on_weak || (unknown && offset != kit->top() && obj != kit->top()))); + + if (!access.is_oop() || !need_read_barrier) { + return load; + } + + if (on_weak) { + // Use the pre-barrier to record the value in the referent field + satb_write_barrier_pre(kit, false /* do_load */, + NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */, + load /* pre_val */, T_OBJECT); + // Add memory barrier to prevent commoning reads from this field + // across safepoint since GC can change its value. + kit->insert_mem_bar(Op_MemBarCPUOrder); + } else if (unknown) { + // We do not require a mem bar inside pre_barrier if need_mem_bar + // is set: the barriers would be emitted by us. + insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar); + } + + return load; +} + +Node* ShenandoahLRBBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const { + GraphKit* kit = access.kit(); + if (access.is_oop()) { + new_val = shenandoah_enqueue_barrier(kit, new_val); + shenandoah_write_barrier_pre(kit, false /* do_load */, + NULL, NULL, max_juint, NULL, NULL, + expected_val /* pre_val */, T_OBJECT); + + MemNode::MemOrd mo = access.mem_node_mo(); + Node* mem = access.memory(); + Node* adr = access.addr().node(); + const TypePtr* adr_type = access.addr().type(); + Node* load_store = NULL; + +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } else +#endif + { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } + + access.set_raw_access(load_store); + pin_atomic_op(access); + +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + } +#endif + load_store = shenandoah_write_barrier(kit, load_store); + return load_store; + } + return ShenandoahBaseBarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); + +} + +Node* ShenandoahLRBBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const { + GraphKit* kit = access.kit(); + if (access.is_oop()) { + new_val = shenandoah_enqueue_barrier(kit, new_val); + shenandoah_write_barrier_pre(kit, false /* do_load */, + NULL, NULL, max_juint, NULL, NULL, + expected_val /* pre_val */, T_OBJECT); + DecoratorSet decorators = access.decorators(); + MemNode::MemOrd mo = access.mem_node_mo(); + Node* mem = access.memory(); + bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0; + Node* load_store = NULL; + Node* adr = access.addr().node(); +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } + } else +#endif + { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } + } + access.set_raw_access(load_store); + pin_atomic_op(access); + return load_store; + } + return ShenandoahBaseBarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); +} + +Node* ShenandoahLRBBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { + GraphKit* kit = access.kit(); + if (access.is_oop()) { + new_val = shenandoah_enqueue_barrier(kit, new_val); + } + Node* result = ShenandoahBaseBarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type); + if (access.is_oop()) { + shenandoah_write_barrier_pre(kit, false /* do_load */, + NULL, NULL, max_juint, NULL, NULL, + result /* pre_val */, T_OBJECT); + result = shenandoah_write_barrier(kit, result); + } + return result; +} diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahLRBBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahLRBBarrierSetC2.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahLRBBarrierSetC2.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHLRBBARRIERSETC2_HPP +#define SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHLRBBARRIERSETC2_HPP + +#include "gc/shenandoah/c2/shenandoahBaseBarrierSetC2.hpp" + +class ShenandoahLRBBarrierSetC2 : public ShenandoahBaseBarrierSetC2 { +protected: + virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const; + virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; + virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* val_type) const; + virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const; + virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const; +}; + +#endif // SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHLRBBARRIERSETC2_HPP diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -423,7 +423,7 @@ bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) { if (UseShenandoahGC) { - if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) { +// if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) { bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; C->clear_major_progress(); PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); @@ -437,7 +437,7 @@ } C->clear_major_progress(); } - } +// } } return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp @@ -192,6 +192,11 @@ FLAG_SET_DEFAULT(NodeLimitFudgeFactor, NodeLimitFudgeFactor * 3); } #endif + + if (ShenandoahLoadRefBarrier) { + // TODO: Disabled because not supported yet. Please implement. + FLAG_SET_DEFAULT(ShenandoahVerifyOptoBarriers, false); + } } size_t ShenandoahArguments::conservative_max_heap_alignment() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -42,47 +42,12 @@ class ShenandoahBarrierSetC1; class ShenandoahBarrierSetC2; -template -class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure { -private: - ShenandoahHeap* _heap; - ShenandoahBarrierSet* _bs; - - template - inline void do_oop_work(T* p) { - oop o; - if (STOREVAL_WRITE_BARRIER) { - o = _heap->evac_update_with_forwarded(p); - if (!CompressedOops::is_null(o)) { - _bs->enqueue(o); - } - } else { - _heap->maybe_update_with_forwarded(p); - } - } -public: - ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()), _bs(ShenandoahBarrierSet::barrier_set()) { - assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); - } - - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_work(p); } -}; - ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : - BarrierSet(make_barrier_set_assembler(), - make_barrier_set_c1(), - make_barrier_set_c2(), - BarrierSet::FakeRtti(BarrierSet::Shenandoah)), - _heap(heap), - _satb_mark_queue_set() -{ -} - -ShenandoahBarrierSetAssembler* ShenandoahBarrierSet::assembler() { - BarrierSetAssembler* const bsa = BarrierSet::barrier_set()->barrier_set_assembler(); - return reinterpret_cast(bsa); -} + ShenandoahBaseBarrierSet(make_barrier_set_assembler(), + make_barrier_set_c1(), + make_barrier_set_c2(), + BarrierSet::FakeRtti(BarrierSet::Shenandoah), + heap) {} void ShenandoahBarrierSet::print_on(outputStream* st) const { st->print("ShenandoahBarrierSet"); @@ -92,77 +57,6 @@ return bsn == BarrierSet::Shenandoah; } -bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) { - return true; -} - -void ShenandoahBarrierSet::resize_covered_region(MemRegion mr) { - Unimplemented(); -} - -void ShenandoahBarrierSet::write_ref_array_work(MemRegion r) { - ShouldNotReachHere(); -} - -template -void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) { - assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); - ShenandoahUpdateRefsForOopClosure cl; - T* dst = (T*) start; - for (size_t i = 0; i < count; i++) { - cl.do_oop(dst++); - } -} - -void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) { - assert(UseShenandoahGC, "should be enabled"); - if (count == 0) return; - if (!ShenandoahCloneBarrier) return; - - if (!need_update_refs_barrier()) return; - - if (_heap->is_concurrent_traversal_in_progress()) { - ShenandoahEvacOOMScope oom_evac_scope; - if (UseCompressedOops) { - write_ref_array_loop(start, count); - } else { - write_ref_array_loop(start, count); - } - } else { - if (UseCompressedOops) { - write_ref_array_loop(start, count); - } else { - write_ref_array_loop(start, count); - } - } -} - -template -void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, size_t count) { - shenandoah_assert_not_in_cset_loc_except(dst, _heap->cancelled_gc()); - if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { - T* elem_ptr = dst; - for (size_t i = 0; i < count; i++, elem_ptr++) { - T heap_oop = RawAccess<>::oop_load(elem_ptr); - if (!CompressedOops::is_null(heap_oop)) { - enqueue(CompressedOops::decode_not_null(heap_oop)); - } - } - } -} - -void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized) { - if (! dest_uninitialized) { - write_ref_array_pre_work(dst, count); - } -} - -void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, size_t count, bool dest_uninitialized) { - if (! dest_uninitialized) { - write_ref_array_pre_work(dst, count); - } -} - template inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop new_val) { shenandoah_assert_not_in_cset_loc_except(field, _heap->cancelled_gc()); @@ -192,193 +86,3 @@ shenandoah_assert_not_forwarded_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress()); shenandoah_assert_not_in_cset_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress()); } - -void ShenandoahBarrierSet::write_region(MemRegion mr) { - assert(UseShenandoahGC, "should be enabled"); - if (!ShenandoahCloneBarrier) return; - if (! need_update_refs_barrier()) return; - - // This is called for cloning an object (see jvm.cpp) after the clone - // has been made. We are not interested in any 'previous value' because - // it would be NULL in any case. But we *are* interested in any oop* - // that potentially need to be updated. - - oop obj = oop(mr.start()); - shenandoah_assert_correct(NULL, obj); - if (_heap->is_concurrent_traversal_in_progress()) { - ShenandoahEvacOOMScope oom_evac_scope; - ShenandoahUpdateRefsForOopClosure cl; - obj->oop_iterate(&cl); - } else { - ShenandoahUpdateRefsForOopClosure cl; - obj->oop_iterate(&cl); - } -} - -oop ShenandoahBarrierSet::read_barrier(oop src) { - // Check for forwarded objects, because on Full GC path we might deal with - // non-trivial fwdptrs that contain Full GC specific metadata. We could check - // for is_full_gc_in_progress(), but this also covers the case of stable heap, - // which provides a bit of performance improvement. - if (ShenandoahReadBarrier && _heap->has_forwarded_objects()) { - return ShenandoahBarrierSet::resolve_forwarded(src); - } else { - return src; - } -} - -bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) { - bool eq = oopDesc::unsafe_equals(obj1, obj2); - if (! eq && ShenandoahAcmpBarrier) { - OrderAccess::loadload(); - obj1 = resolve_forwarded(obj1); - obj2 = resolve_forwarded(obj2); - eq = oopDesc::unsafe_equals(obj1, obj2); - } - return eq; -} - -oop ShenandoahBarrierSet::write_barrier_mutator(oop obj) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); - assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL), "evac should be in progress"); - shenandoah_assert_in_cset(NULL, obj); - - oop fwd = resolve_forwarded_not_null(obj); - if (oopDesc::unsafe_equals(obj, fwd)) { - ShenandoahEvacOOMScope oom_evac_scope; - - Thread* thread = Thread::current(); - oop res_oop = _heap->evacuate_object(obj, thread); - - // Since we are already here and paid the price of getting through runtime call adapters - // and acquiring oom-scope, it makes sense to try and evacuate more adjacent objects, - // thus amortizing the overhead. For sparsely live heaps, scan costs easily dominate - // total assist costs, and can introduce a lot of evacuation latency. This is why we - // only scan for _nearest_ N objects, regardless if they are eligible for evac or not. - // The scan itself should also avoid touching the non-marked objects below TAMS, because - // their metadata (notably, klasses) may be incorrect already. - - size_t max = ShenandoahEvacAssist; - if (max > 0) { - // Traversal is special: it uses incomplete marking context, because it coalesces evac with mark. - // Other code uses complete marking context, because evac happens after the mark. - ShenandoahMarkingContext* ctx = _heap->is_concurrent_traversal_in_progress() ? - _heap->marking_context() : _heap->complete_marking_context(); - - ShenandoahHeapRegion* r = _heap->heap_region_containing(obj); - assert(r->is_cset(), "sanity"); - - HeapWord* cur = (HeapWord*)obj + obj->size() + BrooksPointer::word_size(); - - size_t count = 0; - while ((cur < r->top()) && ctx->is_marked(oop(cur)) && (count++ < max)) { - oop cur_oop = oop(cur); - if (oopDesc::unsafe_equals(cur_oop, resolve_forwarded_not_null(cur_oop))) { - _heap->evacuate_object(cur_oop, thread); - } - cur = cur + cur_oop->size() + BrooksPointer::word_size(); - } - } - - return res_oop; - } - return fwd; -} - -oop ShenandoahBarrierSet::write_barrier_impl(oop obj) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); - if (!CompressedOops::is_null(obj)) { - bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); - oop fwd = resolve_forwarded_not_null(obj); - if (evac_in_progress && - _heap->in_collection_set(obj) && - oopDesc::unsafe_equals(obj, fwd)) { - Thread *t = Thread::current(); - if (t->is_GC_task_thread()) { - return _heap->evacuate_object(obj, t); - } else { - ShenandoahEvacOOMScope oom_evac_scope; - return _heap->evacuate_object(obj, t); - } - } else { - return fwd; - } - } else { - return obj; - } -} - -oop ShenandoahBarrierSet::write_barrier(oop obj) { - if (ShenandoahWriteBarrier) { - return write_barrier_impl(obj); - } else { - return obj; - } -} - -oop ShenandoahBarrierSet::storeval_barrier(oop obj) { - if (ShenandoahStoreValEnqueueBarrier) { - if (!CompressedOops::is_null(obj)) { - obj = write_barrier(obj); - enqueue(obj); - } - } - if (ShenandoahStoreValReadBarrier) { - obj = resolve_forwarded(obj); - } - return obj; -} - -void ShenandoahBarrierSet::keep_alive_barrier(oop obj) { - if (ShenandoahKeepAliveBarrier && _heap->is_concurrent_mark_in_progress()) { - enqueue(obj); - } -} - -void ShenandoahBarrierSet::enqueue(oop obj) { - shenandoah_assert_not_forwarded_if(NULL, obj, ShenandoahHeap::heap()->is_concurrent_traversal_in_progress()); - if (!_satb_mark_queue_set.is_active()) return; - - // Filter marked objects before hitting the SATB queues. The same predicate would - // be used by SATBMQ::filter to eliminate already marked objects downstream, but - // filtering here helps to avoid wasteful SATB queueing work to begin with. - if (!_heap->requires_marking(obj)) return; - - Thread* thr = Thread::current(); - if (thr->is_Java_thread()) { - ShenandoahThreadLocalData::satb_mark_queue(thr).enqueue(obj); - } else { - MutexLockerEx x(Shared_SATB_Q_lock, Mutex::_no_safepoint_check_flag); - _satb_mark_queue_set.shared_satb_queue()->enqueue(obj); - } -} - -void ShenandoahBarrierSet::on_thread_create(Thread* thread) { - // Create thread local data - ShenandoahThreadLocalData::create(thread); -} - -void ShenandoahBarrierSet::on_thread_destroy(Thread* thread) { - // Destroy thread local data - ShenandoahThreadLocalData::destroy(thread); -} - - -void ShenandoahBarrierSet::on_thread_attach(JavaThread* thread) { - assert(!SafepointSynchronize::is_at_safepoint(), "We should not be at a safepoint"); - assert(!ShenandoahThreadLocalData::satb_mark_queue(thread).is_active(), "SATB queue should not be active"); - assert(ShenandoahThreadLocalData::satb_mark_queue(thread).is_empty(), "SATB queue should be empty"); - if (ShenandoahBarrierSet::satb_mark_queue_set().is_active()) { - ShenandoahThreadLocalData::satb_mark_queue(thread).set_active(true); - } - ShenandoahThreadLocalData::set_gc_state(thread, ShenandoahHeap::heap()->gc_state()); - ShenandoahThreadLocalData::initialize_gclab(thread); -} - -void ShenandoahBarrierSet::on_thread_detach(JavaThread* thread) { - ShenandoahThreadLocalData::satb_mark_queue(thread).flush(); - PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); - if (gclab != NULL) { - gclab->retire(); - } -} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -25,56 +25,18 @@ #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP #include "gc/shared/accessBarrierSupport.hpp" -#include "gc/shared/barrierSet.hpp" -#include "gc/shenandoah/shenandoahHeap.hpp" -#include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSet.hpp" -class ShenandoahBarrierSetAssembler; - -class ShenandoahBarrierSet: public BarrierSet { -private: - - enum ArrayCopyStoreValMode { - NONE, - READ_BARRIER, - WRITE_BARRIER - }; - - ShenandoahHeap* _heap; - ShenandoahSATBMarkQueueSet _satb_mark_queue_set; +class ShenandoahBarrierSet: public ShenandoahBaseBarrierSet { public: ShenandoahBarrierSet(ShenandoahHeap* heap); - static ShenandoahBarrierSetAssembler* assembler(); - - inline static ShenandoahBarrierSet* barrier_set() { - return barrier_set_cast(BarrierSet::barrier_set()); - } - - static ShenandoahSATBMarkQueueSet& satb_mark_queue_set() { - return barrier_set()->_satb_mark_queue_set; - } - void print_on(outputStream* st) const; bool is_a(BarrierSet::Name bsn); - bool is_aligned(HeapWord* hw); - void resize_covered_region(MemRegion mr); - - void write_ref_array(HeapWord* start, size_t count); - void write_ref_array_work(MemRegion r); - - template void - write_ref_array_pre_work(T* dst, size_t count); - - void write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized); - - void write_ref_array_pre(narrowOop* dst, size_t count, bool dest_uninitialized); - - // We export this to make it available in cases where the static // type of the barrier set is known. Note that it is non-virtual. template inline void inline_write_ref_field_pre(T* field, oop new_val); @@ -85,64 +47,6 @@ void write_ref_field_pre_work(void* field, oop new_val); void write_ref_field_work(void* v, oop o, bool release = false); - void write_region(MemRegion mr); - - virtual void on_thread_create(Thread* thread); - virtual void on_thread_destroy(Thread* thread); - virtual void on_thread_attach(JavaThread* thread); - virtual void on_thread_detach(JavaThread* thread); - - virtual oop read_barrier(oop src); - - static inline oop resolve_forwarded_not_null(oop p); - static inline oop resolve_forwarded(oop p); - - virtual oop write_barrier(oop obj); - - oop write_barrier_mutator(oop obj); - - virtual oop storeval_barrier(oop obj); - - virtual void keep_alive_barrier(oop obj); - - bool obj_equals(oop obj1, oop obj2); - - void enqueue(oop obj); - -private: - inline bool need_update_refs_barrier(); - - template - void write_ref_array_loop(HeapWord* start, size_t count); - - oop write_barrier_impl(oop obj); - - static void keep_alive_if_weak(DecoratorSet decorators, oop value) { - assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); - const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0; - const bool peek = (decorators & AS_NO_KEEPALIVE) != 0; - if (!peek && !on_strong_oop_ref && value != NULL) { - ShenandoahBarrierSet::barrier_set()->keep_alive_barrier(value); - } - } - - template - bool arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, - bool checkcast, bool satb, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode); - - template - bool arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, - bool satb, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode); - - template - bool arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, - ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode); - - template - bool arraycopy_loop(T* src, T* dst, size_t length, Klass* bound); - - template - bool arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread); public: // Callbacks for runtime accesses. @@ -162,7 +66,7 @@ template static T load_in_heap_at(oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::resolve_forwarded(base); + base = ShenandoahBaseBarrierSet::resolve_forwarded(base); return Raw::template load_at(base, offset); } @@ -174,7 +78,7 @@ template static void store_in_heap_at(oop base, ptrdiff_t offset, T value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); + base = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(base); Raw::store_at(base, offset, value); } @@ -186,7 +90,7 @@ template static T atomic_cmpxchg_in_heap_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); + base = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(base); return Raw::atomic_cmpxchg_at(new_value, base, offset, compare_value); } @@ -198,7 +102,7 @@ template static T atomic_xchg_in_heap_at(T new_value, oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); + base = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(base); return Raw::atomic_xchg_at(new_value, base, offset); } @@ -219,7 +123,7 @@ } static oop oop_load_in_heap_at(oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::resolve_forwarded(base); + base = ShenandoahBaseBarrierSet::resolve_forwarded(base); oop value = Raw::template oop_load_at(base, offset); keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); return value; @@ -227,13 +131,13 @@ template static void oop_store_in_heap(T* addr, oop value) { - ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); + ShenandoahBaseBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); Raw::oop_store(addr, value); } static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); + base = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(base); + value = ShenandoahBaseBarrierSet::barrier_set()->storeval_barrier(value); oop_store_in_heap(AccessInternal::oop_field_addr(base, offset), value); } @@ -242,8 +146,8 @@ static oop oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value); static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); + base = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(base); + new_value = ShenandoahBaseBarrierSet::barrier_set()->storeval_barrier(new_value); return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr(base, offset), compare_value); } @@ -251,8 +155,8 @@ static oop oop_atomic_xchg_in_heap(oop new_value, T* addr); static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); + base = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(base); + new_value = ShenandoahBaseBarrierSet::barrier_set()->storeval_barrier(new_value); return oop_atomic_xchg_in_heap(new_value, AccessInternal::oop_field_addr(base, offset)); } @@ -273,11 +177,11 @@ } static oop resolve(oop obj) { - return ShenandoahBarrierSet::barrier_set()->write_barrier(obj); + return ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj); } static bool equals(oop o1, oop o2) { - return ShenandoahBarrierSet::barrier_set()->obj_equals(o1, o2); + return ShenandoahBaseBarrierSet::barrier_set()->obj_equals(o1, o2); } }; @@ -291,7 +195,7 @@ template<> struct BarrierSet::GetType { - typedef ShenandoahBarrierSet type; + typedef ::ShenandoahBarrierSet type; }; #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -27,26 +27,9 @@ #include "gc/shared/barrierSet.hpp" #include "gc/shenandoah/brooksPointer.inline.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSet.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" -bool ShenandoahBarrierSet::need_update_refs_barrier() { - return _heap->is_update_refs_in_progress() || - _heap->is_concurrent_traversal_in_progress() || - (_heap->is_concurrent_mark_in_progress() && _heap->has_forwarded_objects()); -} - -inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) { - return BrooksPointer::forwardee(p); -} - -inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) { - if (((HeapWord*) p) != NULL) { - return resolve_forwarded_not_null(p); - } else { - return p; - } -} - template template inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { @@ -91,139 +74,6 @@ Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); } -template -bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, - bool checkcast, bool satb, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { - if (checkcast) { - return arraycopy_loop_2(src, dst, length, bound, satb, storeval_mode); - } else { - return arraycopy_loop_2(src, dst, length, bound, satb, storeval_mode); - } -} - -template -bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, - bool satb, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { - if (satb) { - return arraycopy_loop_3(src, dst, length, bound, storeval_mode); - } else { - return arraycopy_loop_3(src, dst, length, bound, storeval_mode); - } -} - -template -bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, - ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { - switch (storeval_mode) { - case NONE: - return arraycopy_loop(src, dst, length, bound); - case READ_BARRIER: - return arraycopy_loop(src, dst, length, bound); - case WRITE_BARRIER: - return arraycopy_loop(src, dst, length, bound); - default: - ShouldNotReachHere(); - return true; // happy compiler - } -} - -template -bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound) { - Thread* thread = Thread::current(); - - ShenandoahEvacOOMScope oom_evac_scope; - - // We need to handle four cases: - // - // a) src < dst, intersecting, can only copy backward only - // [...src...] - // [...dst...] - // - // b) src < dst, non-intersecting, can copy forward/backward - // [...src...] - // [...dst...] - // - // c) src > dst, intersecting, can copy forward only - // [...src...] - // [...dst...] - // - // d) src > dst, non-intersecting, can copy forward/backward - // [...src...] - // [...dst...] - // - if (src > dst) { - // copy forward: - T* cur_src = src; - T* cur_dst = dst; - T* src_end = src + length; - for (; cur_src < src_end; cur_src++, cur_dst++) { - if (!arraycopy_element(cur_src, cur_dst, bound, thread)) { - return false; - } - } - } else { - // copy backward: - T* cur_src = src + length - 1; - T* cur_dst = dst + length - 1; - for (; cur_src >= src; cur_src--, cur_dst--) { - if (!arraycopy_element(cur_src, cur_dst, bound, thread)) { - return false; - } - } - } - return true; -} - -template -bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread) { - T o = RawAccess<>::oop_load(cur_src); - - if (SATB) { - T prev = RawAccess<>::oop_load(cur_dst); - if (!CompressedOops::is_null(prev)) { - oop prev_obj = CompressedOops::decode_not_null(prev); - enqueue(prev_obj); - } - } - - if (!CompressedOops::is_null(o)) { - oop obj = CompressedOops::decode_not_null(o); - - if (CHECKCAST) { - assert(bound != NULL, "need element klass for checkcast"); - if (!oopDesc::is_instanceof_or_null(obj, bound)) { - return false; - } - } - - switch (STOREVAL_MODE) { - case NONE: - break; - case READ_BARRIER: - obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); - break; - case WRITE_BARRIER: - if (_heap->in_collection_set(obj)) { - oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); - if (oopDesc::unsafe_equals(forw, obj)) { - forw = _heap->evacuate_object(forw, thread); - } - obj = forw; - } - enqueue(obj); - break; - default: - ShouldNotReachHere(); - } - - RawAccess::oop_store(cur_dst, obj); - } else { - // Store null. - RawAccess<>::oop_store(cur_dst, o); - } - return true; -} - // Clone barrier support template void ShenandoahBarrierSet::AccessBarrier::clone_in_heap(oop src, oop dst, size_t size) { @@ -241,10 +91,10 @@ size_t length) { ShenandoahHeap* heap = ShenandoahHeap::heap(); if (!CompressedOops::is_null(src_obj)) { - src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj)); + src_obj = arrayOop(ShenandoahBaseBarrierSet::barrier_set()->read_barrier(src_obj)); } if (!CompressedOops::is_null(dst_obj)) { - dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj)); + dst_obj = arrayOop(ShenandoahBaseBarrierSet::barrier_set()->write_barrier(dst_obj)); } bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress(); @@ -273,7 +123,7 @@ dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); Klass* bound = objArrayOop(dst_obj)->element_klass(); - ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); + ShenandoahBaseBarrierSet* bs = ShenandoahBaseBarrierSet::barrier_set(); return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, storeval_mode); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.cpp @@ -0,0 +1,374 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1BarrierSet.hpp" +#include "gc/shenandoah/shenandoahAsserts.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSet.hpp" +#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahCollectorPolicy.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeuristics.hpp" +#include "gc/shenandoah/shenandoahTraversalGC.hpp" +#include "memory/iterator.inline.hpp" +#include "runtime/interfaceSupport.inline.hpp" + +template +class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure { +private: + ShenandoahHeap* _heap; + ShenandoahBaseBarrierSet* _bs; + + template + inline void do_oop_work(T* p) { + oop o; + if (STOREVAL_WRITE_BARRIER) { + o = _heap->evac_update_with_forwarded(p); + if (!CompressedOops::is_null(o)) { + _bs->enqueue(o); + } + } else { + _heap->maybe_update_with_forwarded(p); + } + } +public: + ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()), _bs(ShenandoahBaseBarrierSet::barrier_set()) { + assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); + } + + virtual void do_oop(oop* p) { do_oop_work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } +}; + +ShenandoahBaseBarrierSet::ShenandoahBaseBarrierSet(BarrierSetAssembler* bsasm, + BarrierSetC1* bsc1, + BarrierSetC2* bsc2, + const FakeRtti& fake_rtti, + ShenandoahHeap* heap) : + BarrierSet(bsasm, bsc1, bsc2, fake_rtti), + _satb_mark_queue_set(), + _heap(heap) +{ +} + +ShenandoahBaseBarrierSetAssembler* ShenandoahBaseBarrierSet::assembler() { + BarrierSetAssembler* const bsa = BarrierSet::barrier_set()->barrier_set_assembler(); + return reinterpret_cast(bsa); +} + +bool ShenandoahBaseBarrierSet::is_aligned(HeapWord* hw) { + return true; +} + +void ShenandoahBaseBarrierSet::resize_covered_region(MemRegion mr) { + Unimplemented(); +} + +void ShenandoahBaseBarrierSet::write_ref_array_work(MemRegion r) { + ShouldNotReachHere(); +} + +template +void ShenandoahBaseBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) { + assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); + ShenandoahUpdateRefsForOopClosure cl; + T* dst = (T*) start; + for (size_t i = 0; i < count; i++) { + cl.do_oop(dst++); + } +} + +void ShenandoahBaseBarrierSet::write_ref_array(HeapWord* start, size_t count) { + assert(UseShenandoahGC, "should be enabled"); + if (count == 0) return; + if (!ShenandoahCloneBarrier) return; + + if (!need_update_refs_barrier()) return; + + if (_heap->is_concurrent_traversal_in_progress()) { + ShenandoahEvacOOMScope oom_evac_scope; + if (UseCompressedOops) { + write_ref_array_loop(start, count); + } else { + write_ref_array_loop(start, count); + } + } else { + if (UseCompressedOops) { + write_ref_array_loop(start, count); + } else { + write_ref_array_loop(start, count); + } + } +} + +template +void ShenandoahBaseBarrierSet::write_ref_array_pre_work(T* dst, size_t count) { + shenandoah_assert_not_in_cset_loc_except(dst, _heap->cancelled_gc()); + if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { + T* elem_ptr = dst; + for (size_t i = 0; i < count; i++, elem_ptr++) { + T heap_oop = RawAccess<>::oop_load(elem_ptr); + if (!CompressedOops::is_null(heap_oop)) { + enqueue(CompressedOops::decode_not_null(heap_oop)); + } + } + } +} + +void ShenandoahBaseBarrierSet::write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized) { + if (! dest_uninitialized) { + write_ref_array_pre_work(dst, count); + } +} + +void ShenandoahBaseBarrierSet::write_ref_array_pre(narrowOop* dst, size_t count, bool dest_uninitialized) { + if (! dest_uninitialized) { + write_ref_array_pre_work(dst, count); + } +} + +template +inline void ShenandoahBaseBarrierSet::inline_write_ref_field_pre(T* field, oop new_val) { + shenandoah_assert_not_in_cset_loc_except(field, _heap->cancelled_gc()); + if (_heap->is_concurrent_mark_in_progress()) { + T heap_oop = RawAccess<>::oop_load(field); + if (!CompressedOops::is_null(heap_oop)) { + enqueue(CompressedOops::decode(heap_oop)); + } + } +} + +// These are the more general virtual versions. +void ShenandoahBaseBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) { + inline_write_ref_field_pre(field, new_val); +} + +void ShenandoahBaseBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) { + inline_write_ref_field_pre(field, new_val); +} + +void ShenandoahBaseBarrierSet::write_ref_field_pre_work(void* field, oop new_val) { + guarantee(false, "Not needed"); +} + +void ShenandoahBaseBarrierSet::write_ref_field_work(void* v, oop o, bool release) { + shenandoah_assert_not_in_cset_loc_except(v, _heap->cancelled_gc()); + shenandoah_assert_not_forwarded_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress()); + shenandoah_assert_not_in_cset_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress()); +} + +void ShenandoahBaseBarrierSet::write_region(MemRegion mr) { + assert(UseShenandoahGC, "should be enabled"); + if (!ShenandoahCloneBarrier) return; + if (! need_update_refs_barrier()) return; + + // This is called for cloning an object (see jvm.cpp) after the clone + // has been made. We are not interested in any 'previous value' because + // it would be NULL in any case. But we *are* interested in any oop* + // that potentially need to be updated. + + oop obj = oop(mr.start()); + shenandoah_assert_correct(NULL, obj); + if (_heap->is_concurrent_traversal_in_progress()) { + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } else { + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } +} + +oop ShenandoahBaseBarrierSet::read_barrier(oop src) { + // Check for forwarded objects, because on Full GC path we might deal with + // non-trivial fwdptrs that contain Full GC specific metadata. We could check + // for is_full_gc_in_progress(), but this also covers the case of stable heap, + // which provides a bit of performance improvement. + if (ShenandoahReadBarrier && _heap->has_forwarded_objects()) { + return ShenandoahBarrierSet::resolve_forwarded(src); + } else { + return src; + } +} + +bool ShenandoahBaseBarrierSet::obj_equals(oop obj1, oop obj2) { + bool eq = oopDesc::unsafe_equals(obj1, obj2); + if (! eq && ShenandoahAcmpBarrier) { + OrderAccess::loadload(); + obj1 = resolve_forwarded(obj1); + obj2 = resolve_forwarded(obj2); + eq = oopDesc::unsafe_equals(obj1, obj2); + } + return eq; +} + +oop ShenandoahBaseBarrierSet::write_barrier_mutator(oop obj) { + assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); + assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL), "evac should be in progress"); + shenandoah_assert_in_cset(NULL, obj); + + oop fwd = resolve_forwarded_not_null(obj); + if (oopDesc::unsafe_equals(obj, fwd)) { + ShenandoahEvacOOMScope oom_evac_scope; + + Thread* thread = Thread::current(); + oop res_oop = _heap->evacuate_object(obj, thread); + + // Since we are already here and paid the price of getting through runtime call adapters + // and acquiring oom-scope, it makes sense to try and evacuate more adjacent objects, + // thus amortizing the overhead. For sparsely live heaps, scan costs easily dominate + // total assist costs, and can introduce a lot of evacuation latency. This is why we + // only scan for _nearest_ N objects, regardless if they are eligible for evac or not. + // The scan itself should also avoid touching the non-marked objects below TAMS, because + // their metadata (notably, klasses) may be incorrect already. + + size_t max = ShenandoahEvacAssist; + if (max > 0) { + // Traversal is special: it uses incomplete marking context, because it coalesces evac with mark. + // Other code uses complete marking context, because evac happens after the mark. + ShenandoahMarkingContext* ctx = _heap->is_concurrent_traversal_in_progress() ? + _heap->marking_context() : _heap->complete_marking_context(); + + ShenandoahHeapRegion* r = _heap->heap_region_containing(obj); + assert(r->is_cset(), "sanity"); + + HeapWord* cur = (HeapWord*)obj + obj->size() + BrooksPointer::word_size(); + + size_t count = 0; + while ((cur < r->top()) && ctx->is_marked(oop(cur)) && (count++ < max)) { + oop cur_oop = oop(cur); + if (oopDesc::unsafe_equals(cur_oop, resolve_forwarded_not_null(cur_oop))) { + _heap->evacuate_object(cur_oop, thread); + } + cur = cur + cur_oop->size() + BrooksPointer::word_size(); + } + } + + return res_oop; + } + return fwd; +} + +oop ShenandoahBaseBarrierSet::write_barrier_impl(oop obj) { + assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); + if (!CompressedOops::is_null(obj)) { + bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); + oop fwd = resolve_forwarded_not_null(obj); + if (evac_in_progress && + _heap->in_collection_set(obj) && + oopDesc::unsafe_equals(obj, fwd)) { + Thread *t = Thread::current(); + if (t->is_GC_task_thread()) { + return _heap->evacuate_object(obj, t); + } else { + ShenandoahEvacOOMScope oom_evac_scope; + return _heap->evacuate_object(obj, t); + } + } else { + return fwd; + } + } else { + return obj; + } +} + +oop ShenandoahBaseBarrierSet::write_barrier(oop obj) { + if (ShenandoahWriteBarrier) { + return write_barrier_impl(obj); + } else { + return obj; + } +} + +oop ShenandoahBaseBarrierSet::storeval_barrier(oop obj) { + if (ShenandoahStoreValEnqueueBarrier) { + if (!CompressedOops::is_null(obj)) { + obj = write_barrier(obj); + enqueue_barrier(obj); + } + } + if (ShenandoahStoreValReadBarrier) { + obj = resolve_forwarded(obj); + } + return obj; +} + +void ShenandoahBaseBarrierSet::enqueue_barrier(oop obj) { + if (ShenandoahStoreValEnqueueBarrier && obj != NULL) { + enqueue(obj); + } +} + +void ShenandoahBaseBarrierSet::keep_alive_barrier(oop obj) { + if (ShenandoahKeepAliveBarrier && _heap->is_concurrent_mark_in_progress()) { + enqueue(obj); + } +} + +void ShenandoahBaseBarrierSet::enqueue(oop obj) { + shenandoah_assert_not_forwarded_if(NULL, obj, ShenandoahHeap::heap()->is_concurrent_traversal_in_progress()); + if (!_satb_mark_queue_set.is_active()) return; + + // Filter marked objects before hitting the SATB queues. The same predicate would + // be used by SATBMQ::filter to eliminate already marked objects downstream, but + // filtering here helps to avoid wasteful SATB queueing work to begin with. + if (!_heap->requires_marking(obj)) return; + + Thread* thr = Thread::current(); + if (thr->is_Java_thread()) { + ShenandoahThreadLocalData::satb_mark_queue(thr).enqueue(obj); + } else { + MutexLockerEx x(Shared_SATB_Q_lock, Mutex::_no_safepoint_check_flag); + _satb_mark_queue_set.shared_satb_queue()->enqueue(obj); + } +} + +void ShenandoahBaseBarrierSet::on_thread_create(Thread* thread) { + // Create thread local data + ShenandoahThreadLocalData::create(thread); +} + +void ShenandoahBaseBarrierSet::on_thread_destroy(Thread* thread) { + // Destroy thread local data + ShenandoahThreadLocalData::destroy(thread); +} + + +void ShenandoahBaseBarrierSet::on_thread_attach(JavaThread* thread) { + assert(!SafepointSynchronize::is_at_safepoint(), "We should not be at a safepoint"); + assert(!ShenandoahThreadLocalData::satb_mark_queue(thread).is_active(), "SATB queue should not be active"); + assert(ShenandoahThreadLocalData::satb_mark_queue(thread).is_empty(), "SATB queue should be empty"); + if (ShenandoahBarrierSet::satb_mark_queue_set().is_active()) { + ShenandoahThreadLocalData::satb_mark_queue(thread).set_active(true); + } + ShenandoahThreadLocalData::set_gc_state(thread, ShenandoahHeap::heap()->gc_state()); + ShenandoahThreadLocalData::initialize_gclab(thread); +} + +void ShenandoahBaseBarrierSet::on_thread_detach(JavaThread* thread) { + ShenandoahThreadLocalData::satb_mark_queue(thread).flush(); + PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); + if (gclab != NULL) { + gclab->retire(); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.hpp @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSET_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSET_HPP + +#include "gc/shared/barrierSet.hpp" +#include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp" + +class BarrierSetAssembler; +class BarrierSetC1; +class BarrierSetC2; +class ShenandoahBaseBarrierSetAssembler; + +class ShenandoahBaseBarrierSet: public BarrierSet { +private: + + enum ArrayCopyStoreValMode { + NONE, + READ_BARRIER, + WRITE_BARRIER + }; + + ShenandoahSATBMarkQueueSet _satb_mark_queue_set; + +protected: + ShenandoahHeap* _heap; + +public: + + ShenandoahBaseBarrierSet(BarrierSetAssembler* bsasm, + BarrierSetC1* bsc1, + BarrierSetC2* bsc2, + const FakeRtti& fake_rtti, + ShenandoahHeap* heap); + + static ShenandoahBaseBarrierSetAssembler* assembler(); + + inline static ShenandoahBaseBarrierSet* barrier_set() { + //return barrier_set_cast(BarrierSet::barrier_set()); + return reinterpret_cast(BarrierSet::barrier_set()); + } + + static ShenandoahSATBMarkQueueSet& satb_mark_queue_set() { + return barrier_set()->_satb_mark_queue_set; + } + + bool is_aligned(HeapWord* hw); + void resize_covered_region(MemRegion mr); + + void write_ref_array(HeapWord* start, size_t count); + void write_ref_array_work(MemRegion r); + + template void + write_ref_array_pre_work(T* dst, size_t count); + + void write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized); + + void write_ref_array_pre(narrowOop* dst, size_t count, bool dest_uninitialized); + + + // We export this to make it available in cases where the static + // type of the barrier set is known. Note that it is non-virtual. + template inline void inline_write_ref_field_pre(T* field, oop new_val); + + // These are the more general virtual versions. + void write_ref_field_pre_work(oop* field, oop new_val); + void write_ref_field_pre_work(narrowOop* field, oop new_val); + void write_ref_field_pre_work(void* field, oop new_val); + + void write_ref_field_work(void* v, oop o, bool release = false); + void write_region(MemRegion mr); + + virtual void on_thread_create(Thread* thread); + virtual void on_thread_destroy(Thread* thread); + virtual void on_thread_attach(JavaThread* thread); + virtual void on_thread_detach(JavaThread* thread); + + virtual oop read_barrier(oop src); + + static inline oop resolve_forwarded_not_null(oop p); + static inline oop resolve_forwarded(oop p); + + virtual oop write_barrier(oop obj); + + oop write_barrier_mutator(oop obj); + + virtual oop storeval_barrier(oop obj); + virtual void enqueue_barrier(oop obj); + + virtual void keep_alive_barrier(oop obj); + + bool obj_equals(oop obj1, oop obj2); + + void enqueue(oop obj); + + template + bool arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, + bool checkcast, bool satb, ShenandoahBaseBarrierSet::ArrayCopyStoreValMode storeval_mode); + + +private: + inline bool need_update_refs_barrier(); + + template + void write_ref_array_loop(HeapWord* start, size_t count); + + oop write_barrier_impl(oop obj); + +protected: + static void keep_alive_if_weak(DecoratorSet decorators, oop value) { + assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); + const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0; + const bool peek = (decorators & AS_NO_KEEPALIVE) != 0; + if (!peek && !on_strong_oop_ref && value != NULL) { + ShenandoahBaseBarrierSet::barrier_set()->keep_alive_barrier(value); + } + } + +private: + template + bool arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, + bool satb, ShenandoahBaseBarrierSet::ArrayCopyStoreValMode storeval_mode); + + template + bool arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, + ShenandoahBaseBarrierSet::ArrayCopyStoreValMode storeval_mode); + + template + bool arraycopy_loop(T* src, T* dst, size_t length, Klass* bound); + + template + bool arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread); + +}; + +template<> +struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::ShenandoahBase; +}; + +#endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.inline.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSet.inline.hpp @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSET_INLINE_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSET_INLINE_HPP + +#include "gc/shared/barrierSet.hpp" +#include "gc/shenandoah/brooksPointer.inline.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSet.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" + +bool ShenandoahBaseBarrierSet::need_update_refs_barrier() { + return _heap->is_update_refs_in_progress() || + _heap->is_concurrent_traversal_in_progress() || + (_heap->is_concurrent_mark_in_progress() && _heap->has_forwarded_objects()); +} + +inline oop ShenandoahBaseBarrierSet::resolve_forwarded_not_null(oop p) { + return BrooksPointer::forwardee(p); +} + +inline oop ShenandoahBaseBarrierSet::resolve_forwarded(oop p) { + if (((HeapWord*) p) != NULL) { + return resolve_forwarded_not_null(p); + } else { + return p; + } +} + + +template +bool ShenandoahBaseBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, + bool checkcast, bool satb, ShenandoahBaseBarrierSet::ArrayCopyStoreValMode storeval_mode) { + if (checkcast) { + return arraycopy_loop_2(src, dst, length, bound, satb, storeval_mode); + } else { + return arraycopy_loop_2(src, dst, length, bound, satb, storeval_mode); + } +} + +template +bool ShenandoahBaseBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, + bool satb, ShenandoahBaseBarrierSet::ArrayCopyStoreValMode storeval_mode) { + if (satb) { + return arraycopy_loop_3(src, dst, length, bound, storeval_mode); + } else { + return arraycopy_loop_3(src, dst, length, bound, storeval_mode); + } +} + +template +bool ShenandoahBaseBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, + ShenandoahBaseBarrierSet::ArrayCopyStoreValMode storeval_mode) { + switch (storeval_mode) { + case NONE: + return arraycopy_loop(src, dst, length, bound); + case READ_BARRIER: + return arraycopy_loop(src, dst, length, bound); + case WRITE_BARRIER: + return arraycopy_loop(src, dst, length, bound); + default: + ShouldNotReachHere(); + return true; // happy compiler + } +} + +template +bool ShenandoahBaseBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound) { + Thread* thread = Thread::current(); + + ShenandoahEvacOOMScope oom_evac_scope; + + // We need to handle four cases: + // + // a) src < dst, intersecting, can only copy backward only + // [...src...] + // [...dst...] + // + // b) src < dst, non-intersecting, can copy forward/backward + // [...src...] + // [...dst...] + // + // c) src > dst, intersecting, can copy forward only + // [...src...] + // [...dst...] + // + // d) src > dst, non-intersecting, can copy forward/backward + // [...src...] + // [...dst...] + // + if (src > dst) { + // copy forward: + T* cur_src = src; + T* cur_dst = dst; + T* src_end = src + length; + for (; cur_src < src_end; cur_src++, cur_dst++) { + if (!arraycopy_element(cur_src, cur_dst, bound, thread)) { + return false; + } + } + } else { + // copy backward: + T* cur_src = src + length - 1; + T* cur_dst = dst + length - 1; + for (; cur_src >= src; cur_src--, cur_dst--) { + if (!arraycopy_element(cur_src, cur_dst, bound, thread)) { + return false; + } + } + } + return true; +} + +template +bool ShenandoahBaseBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread) { + T o = RawAccess<>::oop_load(cur_src); + + if (SATB) { + T prev = RawAccess<>::oop_load(cur_dst); + if (!CompressedOops::is_null(prev)) { + oop prev_obj = CompressedOops::decode_not_null(prev); + enqueue(prev_obj); + } + } + + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + + if (CHECKCAST) { + assert(bound != NULL, "need element klass for checkcast"); + if (!oopDesc::is_instanceof_or_null(obj, bound)) { + return false; + } + } + + switch (STOREVAL_MODE) { + case NONE: + break; + case READ_BARRIER: + obj = ShenandoahBaseBarrierSet::resolve_forwarded_not_null(obj); + break; + case WRITE_BARRIER: + if (_heap->in_collection_set(obj)) { + oop forw = ShenandoahBaseBarrierSet::resolve_forwarded_not_null(obj); + if (oopDesc::unsafe_equals(forw, obj)) { + forw = _heap->evacuate_object(forw, thread); + } + obj = forw; + } + enqueue(obj); + break; + default: + ShouldNotReachHere(); + } + + RawAccess::oop_store(cur_dst, obj); + } else { + // Store null. + RawAccess<>::oop_store(cur_dst, o); + } + return true; +} + +#endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSET_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetAssembler.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSetAssembler.cpp rename from src/hotspot/share/gc/shenandoah/shenandoahBarrierSetAssembler.cpp rename to src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSetAssembler.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetAssembler.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSetAssembler.cpp @@ -22,13 +22,12 @@ */ #include "precompiled.hpp" -#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp" -bool ShenandoahBarrierSetAssembler::is_shenandoah_wb_C_call(address call) { +bool ShenandoahBaseBarrierSetAssembler::is_shenandoah_wb_C_call(address call) { if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) { return call == _shenandoah_wb_C; } else { return false; } } - diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_HPP + +#include "utilities/macros.hpp" + +#include CPU_HEADER(gc/shenandoah/shenandoahBaseBarrierSetAssembler) + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHBASEBARRIERSETASSEMBLER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -384,7 +384,11 @@ log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads); log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial"); - BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this)); + if (ShenandoahLoadRefBarrier) { + BarrierSet::set_barrier_set(new ShenandoahLRBBarrierSet(this)); + } else { + BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this)); + } _max_workers = MAX2(_max_workers, 1U); _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers, @@ -1126,6 +1130,26 @@ } }; +class ShenandoahEvacuateUpdateAllRootsTask : public AbstractGangTask { + ShenandoahRootProcessor* _rp; +public: + + ShenandoahEvacuateUpdateAllRootsTask(ShenandoahRootProcessor* rp) : + AbstractGangTask("Shenandoah evacuate and update all roots"), + _rp(rp) { + // Nothing else to do. + } + + void work(uint worker_id) { + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahEvacuateUpdateRootsClosure cl; + CLDToOopClosure cldCl(&cl); + // MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations); + CodeBlobToOopClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations); + _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id); + } +}; + class ShenandoahFixRootsTask : public AbstractGangTask { ShenandoahRootEvacuator* _rp; public: @@ -1147,6 +1171,26 @@ } }; +class ShenandoahFixAllRootsTask : public AbstractGangTask { + ShenandoahRootProcessor* _rp; +public: + + ShenandoahFixAllRootsTask(ShenandoahRootProcessor* rp) : + AbstractGangTask("Shenandoah update all roots"), + _rp(rp) + { + // Nothing else to do. + } + + void work(uint worker_id) { + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahUpdateRefsClosure cl; + CLDToOopClosure cldCl(&cl); + MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations); + _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id); + } +}; + void ShenandoahHeap::evacuate_and_update_roots() { #if defined(COMPILER2) || INCLUDE_JVMCI @@ -1154,7 +1198,11 @@ #endif assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped"); - { + if (ShenandoahLoadRefBarrier) { + ShenandoahRootProcessor rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac); + ShenandoahEvacuateUpdateAllRootsTask roots_task(&rp); + workers()->run_task(&roots_task); + } else { ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac); ShenandoahEvacuateUpdateRootsTask roots_task(&rp); workers()->run_task(&roots_task); @@ -1181,9 +1229,15 @@ #if defined(COMPILER2) || INCLUDE_JVMCI DerivedPointerTable::clear(); #endif - ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac); - ShenandoahFixRootsTask update_roots_task(&rp); - workers()->run_task(&update_roots_task); + if (ShenandoahLoadRefBarrier) { + ShenandoahRootProcessor rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac); + ShenandoahFixAllRootsTask update_roots_task(&rp); + workers()->run_task(&update_roots_task); + } else { + ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac); + ShenandoahFixRootsTask update_roots_task(&rp); + workers()->run_task(&update_roots_task); + } #if defined(COMPILER2) || INCLUDE_JVMCI DerivedPointerTable::update_pointers(); #endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/shenandoahLRBBarrierSet.hpp" +#include "gc/shenandoah/shenandoahLRBBarrierSetAssembler.hpp" +#include "gc/shenandoah/c1/shenandoahLRBBarrierSetC1.hpp" +#include "gc/shenandoah/c2/shenandoahLRBBarrierSetC2.hpp" + +ShenandoahLRBBarrierSet::ShenandoahLRBBarrierSet(ShenandoahHeap* heap) : + ShenandoahBaseBarrierSet(make_barrier_set_assembler(), + make_barrier_set_c1(), + make_barrier_set_c2(), + BarrierSet::FakeRtti(BarrierSet::ShenandoahLRB), + heap) {} + +void ShenandoahLRBBarrierSet::print_on(outputStream* st) const { + st->print("ShenandoahLRBBarrierSet"); +} + +bool ShenandoahLRBBarrierSet::is_a(BarrierSet::Name bsn) { + return bsn == BarrierSet::ShenandoahLRB; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.hpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_HPP + +#include "gc/shared/accessBarrierSupport.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSet.hpp" + +class ShenandoahLRBBarrierSet: public ShenandoahBaseBarrierSet { +public: + ShenandoahLRBBarrierSet(ShenandoahHeap* heap); + + void print_on(outputStream* st) const; + bool is_a(BarrierSet::Name bsn); + +public: + // Callbacks for runtime accesses. + template + class AccessBarrier : public BarrierSet::AccessBarrier { + typedef BarrierSet::AccessBarrier Raw; + + public: + template + static oop oop_load_in_heap(T* addr); + static oop oop_load_in_heap_at(oop base, ptrdiff_t offset); + + template + static void oop_store_in_heap(T* addr, oop value); + static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { + oop_store_in_heap(AccessInternal::oop_field_addr(base, offset), value); + } + + template + static oop oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value); + static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value); + + template + static oop oop_atomic_xchg_in_heap(oop new_value, T* addr); + static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset); + + template + static oop oop_load_not_in_heap(T* addr); + + template + static oop oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value); + + template + static oop oop_atomic_xchg_not_in_heap(oop new_value, T* addr); + + template + static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, + size_t length); + + // Clone barrier support + static void clone_in_heap(oop src, oop dst, size_t size); + }; +}; + + +template<> +struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::ShenandoahLRB; +}; + +template<> +struct BarrierSet::GetType { + typedef ::ShenandoahLRBBarrierSet type; +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.inline.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSet.inline.hpp @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_INLINE_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_INLINE_HPP + +#include "gc/shared/accessBarrierSupport.hpp" +#include "gc/shenandoah/shenandoahLRBBarrierSet.hpp" + +template +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_load_in_heap(T* addr) { + oop obj = Raw::oop_load_in_heap(addr); + obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj); + ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj); + return obj; +} + +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_load_in_heap_at(oop base, ptrdiff_t offset) { + oop obj = Raw::oop_load_in_heap_at(base, offset); + obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), obj); + return obj; +} + +template +template +inline void ShenandoahLRBBarrierSet::AccessBarrier::oop_store_in_heap(T* addr, oop value) { + ShenandoahBaseBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); + ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(value); + Raw::oop_store_in_heap(addr, value); +} + +template +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { + ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(new_value); + + oop res; + oop expected = compare_value; + do { + compare_value = expected; + res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); + expected = res; + } while ((! oopDesc::unsafe_equals(compare_value, expected)) && oopDesc::unsafe_equals(resolve_forwarded(compare_value), resolve_forwarded(expected))); + if (oopDesc::unsafe_equals(expected, compare_value)) { + if (ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) { + ShenandoahBarrierSet::barrier_set()->enqueue(compare_value); + } + } + + res = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(res); + ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, res); + return res; +} + +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { + ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(new_value); + + oop res; + oop expected = compare_value; + do { + compare_value = expected; + res = Raw::oop_atomic_cmpxchg_at(new_value, base, offset, compare_value); + expected = res; + } while ((! oopDesc::unsafe_equals(compare_value, expected)) && oopDesc::unsafe_equals(resolve_forwarded(compare_value), resolve_forwarded(expected))); + if (oopDesc::unsafe_equals(expected, compare_value)) { + if (ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) { + ShenandoahBarrierSet::barrier_set()->enqueue(compare_value); + } + } + + res = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(res); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), res); + return res; +} + +template +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap(oop new_value, T* addr) { + ShenandoahBaseBarrierSet::barrier_set()->write_ref_field_pre_work(addr, new_value); + ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(new_value); + oop obj = Raw::oop_atomic_xchg_in_heap(new_value, addr); + obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj); + ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj); + return obj; +} + +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { + ShenandoahBaseBarrierSet::barrier_set()->write_ref_field_pre_work(AccessInternal::oop_field_addr(base, offset), new_value); + ShenandoahBaseBarrierSet::barrier_set()->enqueue_barrier(new_value); + oop obj = Raw::oop_atomic_xchg_in_heap_at(new_value, base, offset); + obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), obj); + return obj; +} + +template +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_load_not_in_heap(T* addr) { + oop obj = Raw::oop_load_not_in_heap(addr); + obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj); + ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj); + return obj; +} + +template +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) { + oop obj = Raw::oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value); + obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj); + ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj); + return obj; +} + +template +template +inline oop ShenandoahLRBBarrierSet::AccessBarrier::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) { + oop obj = Raw::oop_atomic_xchg_not_in_heap(new_value, addr); + obj = ShenandoahBaseBarrierSet::barrier_set()->write_barrier(obj); + ShenandoahBaseBarrierSet::keep_alive_if_weak(decorators, obj); + return obj; +} + +template +template +bool ShenandoahLRBBarrierSet::AccessBarrier::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, + size_t length) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress(); + bool checkcast = HasDecorator::value; + ArrayCopyStoreValMode storeval_mode; + if (heap->has_forwarded_objects()) { + if (heap->is_concurrent_traversal_in_progress()) { + storeval_mode = WRITE_BARRIER; + } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) { + storeval_mode = READ_BARRIER; + } else { + assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress"); + storeval_mode = NONE; // E.g. during evac or outside cycle + } + } else { + assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress"); + storeval_mode = NONE; + } + + if (!satb && !checkcast && storeval_mode == NONE) { + // Short-circuit to bulk copy. + return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); + } + + src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); + dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); + + Klass* bound = objArrayOop(dst_obj)->element_klass(); + ShenandoahBaseBarrierSet* bs = ShenandoahBaseBarrierSet::barrier_set(); + return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, storeval_mode); +} + +template +void ShenandoahLRBBarrierSet::AccessBarrier::clone_in_heap(oop src, oop dst, size_t size) { + Raw::clone(src, dst, size); + ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size)); +} + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSET_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSetAssembler.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSetAssembler.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahLRBBarrierSetAssembler.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSETASSEMBLER_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSETASSEMBLER_HPP + +#include "utilities/macros.hpp" + +#include CPU_HEADER(gc/shenandoah/shenandoahLRBBarrierSetAssembler) + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHLRBBARRIERSETASSEMBLER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp @@ -22,25 +22,25 @@ */ #include "precompiled.hpp" -#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSet.hpp" #include "gc/shenandoah/shenandoahRuntime.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "oops/oop.inline.hpp" void ShenandoahRuntime::write_ref_array_pre_oop_entry(oop* dst, size_t length) { - ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); + ShenandoahBaseBarrierSet *bs = ShenandoahBaseBarrierSet::barrier_set(); bs->write_ref_array_pre(dst, length, false); } void ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry(narrowOop* dst, size_t length) { - ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); + ShenandoahBaseBarrierSet *bs = ShenandoahBaseBarrierSet::barrier_set(); bs->write_ref_array_pre(dst, length, false); } void ShenandoahRuntime::write_ref_array_post_entry(HeapWord* dst, size_t length) { - ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); - bs->ShenandoahBarrierSet::write_ref_array(dst, length); + ShenandoahBaseBarrierSet *bs = ShenandoahBaseBarrierSet::barrier_set(); + bs->ShenandoahBaseBarrierSet::write_ref_array(dst, length); } // Shenandoah pre write barrier slowpath diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -318,6 +318,9 @@ diagnostic(bool, ShenandoahCloneBarrier, true, \ "Turn on/off clone barriers in Shenandoah") \ \ + diagnostic(bool, ShenandoahLoadRefBarrier, true, \ + "Turn on/off load-ref-style barriers") \ + \ diagnostic(bool, ShenandoahStoreCheck, false, \ "Emit additional code that checks objects are written to only" \ " in to-space") \ diff --git a/src/hotspot/share/oops/accessBackend.hpp b/src/hotspot/share/oops/accessBackend.hpp --- a/src/hotspot/share/oops/accessBackend.hpp +++ b/src/hotspot/share/oops/accessBackend.hpp @@ -410,7 +410,13 @@ static oop resolve(oop obj) { return obj; } - static bool equals(oop o1, oop o2) { return o1 == o2; } + static bool equals(oop o1, oop o2) { +#ifndef CHECK_UNHANDLED_OOPS + return o1 == o2; +#else + return o1.obj() == o2.obj(); +#endif + } }; // Below is the implementation of the first 4 steps of the template pipeline: diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -40,6 +40,9 @@ #include "opto/runtime.hpp" #include "opto/subnode.hpp" #include "runtime/sharedRuntime.hpp" +#if INCLUDE_SHENANDOAHGC +#include "gc/shenandoah/c2/shenandoahSupport.hpp" +#endif // Utility function. const TypeFunc* CallGenerator::tf() const { @@ -834,6 +837,9 @@ { // Get MethodHandle receiver: Node* receiver = kit.argument(0); + #if INCLUDE_SHENANDOAHGC + receiver = ShenandoahBarrierNode::skip_through_barrier(receiver); + #endif if (receiver->Opcode() == Op_ConP) { input_not_const = false; const TypeOopPtr* oop_ptr = receiver->bottom_type()->is_oopptr(); @@ -861,11 +867,15 @@ case vmIntrinsics::_linkToVirtual: case vmIntrinsics::_linkToStatic: + case vmIntrinsics::_linkToSpecial: case vmIntrinsics::_linkToInterface: { // Get MemberName argument: Node* member_name = kit.argument(callee->arg_size() - 1); + #if INCLUDE_SHENANDOAHGC + member_name = ShenandoahBarrierNode::skip_through_barrier(member_name); + #endif if (member_name->Opcode() == Op_ConP) { input_not_const = false; const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr(); diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -36,7 +36,7 @@ #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" #if INCLUDE_SHENANDOAHGC -#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahBaseBarrierSetAssembler.hpp" #endif // Optimization - Graph Style @@ -898,7 +898,7 @@ #if INCLUDE_SHENANDOAHGC if (UseShenandoahGC && - ShenandoahBarrierSetAssembler::is_shenandoah_wb_C_call(mcall->entry_point())) { + ShenandoahBaseBarrierSetAssembler::is_shenandoah_wb_C_call(mcall->entry_point())) { assert(op == Op_CallLeafNoFP, "shenandoah_wb_C should be called with Op_CallLeafNoFP"); add_call_kills(proj, regs, save_policy, exclude_soe, true); } else diff --git a/src/hotspot/share/opto/replacednodes.cpp b/src/hotspot/share/opto/replacednodes.cpp --- a/src/hotspot/share/opto/replacednodes.cpp +++ b/src/hotspot/share/opto/replacednodes.cpp @@ -151,7 +151,7 @@ if (use->outcnt() == 0) { continue; } - if (n->is_CFG() || (n->in(0) != NULL && !n->in(0)->is_top())) { + if (n->is_CFG() || (n->in(0) != NULL && !n->in(0)->is_top()) && n->Opcode() != Op_ShenandoahWBMemProj) { int depth = 0; Node *m = n; if (!n->is_CFG()) { diff --git a/src/hotspot/share/opto/stringopts.cpp b/src/hotspot/share/opto/stringopts.cpp --- a/src/hotspot/share/opto/stringopts.cpp +++ b/src/hotspot/share/opto/stringopts.cpp @@ -603,6 +603,8 @@ // Collect the types needed to talk about the various slices of memory byte_adr_idx = C->get_alias_index(TypeAryPtr::BYTES); + const Type* ary_t = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::BYTE, TypeInt::POS), ciTypeArrayKlass::make(T_BYTE), true, 0); + byte_adr_idx_wb = C->get_alias_index(ShenandoahBarrierNode::brooks_pointer_type(ary_t)); // For each locally allocated StringBuffer see if the usages can be // collapsed into a single String construction. @@ -1244,7 +1246,7 @@ } // Simplified version of Integer.getChars -void PhaseStringOpts::getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicType bt, Node* end, Node* final_merge, Node* final_mem, int merge_index) { +void PhaseStringOpts::getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicType bt, Node* end, Node* final_merge, Node* final_mem, Node* final_mem_wb, int merge_index) { // if (i < 0) { // sign = '-'; // i = -i; @@ -1335,11 +1337,13 @@ final_merge->init_req(merge_index + 2, __ IfFalse(iff)); final_mem->init_req(merge_index + 2, kit.memory(byte_adr_idx)); + final_mem_wb->init_req(merge_index + 2, kit.memory(byte_adr_idx_wb)); kit.set_control(__ IfTrue(iff)); if (kit.stopped()) { final_merge->init_req(merge_index + 1, C->top()); final_mem->init_req(merge_index + 1, C->top()); + final_mem_wb->init_req(merge_index + 1, C->top()); } else { Node* index = __ SubI(charPos, __ intcon((bt == T_BYTE) ? 1 : 2)); st = __ store_to_memory(kit.control(), kit.array_element_address(dst_array, index, T_BYTE), @@ -1347,6 +1351,7 @@ final_merge->init_req(merge_index + 1, kit.control()); final_mem->init_req(merge_index + 1, st); + final_mem_wb->init_req(merge_index + 1, kit.memory(byte_adr_idx_wb)); } } @@ -1365,12 +1370,16 @@ Node* final_mem = PhiNode::make(final_merge, kit.memory(byte_adr_idx), Type::MEMORY, TypeAryPtr::BYTES); kit.gvn().set_type(final_mem, Type::MEMORY); + const Type* ary_t = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::BYTE, TypeInt::POS), ciTypeArrayKlass::make(T_BYTE), true, 0); + Node* final_mem_wb = PhiNode::make(final_merge, kit.memory(byte_adr_idx_wb), Type::MEMORY, ShenandoahBarrierNode::brooks_pointer_type(ary_t)); + kit.gvn().set_type(final_mem_wb, Type::MEMORY); // need to handle arg == Integer.MIN_VALUE specially because negating doesn't make it positive IfNode* iff = kit.create_and_map_if(kit.control(), __ Bool(__ CmpI(arg, __ intcon(0x80000000)), BoolTest::ne), PROB_FAIR, COUNT_UNKNOWN); Node* old_mem = kit.memory(byte_adr_idx); + Node* old_mem_wb = kit.memory(byte_adr_idx_wb); kit.set_control(__ IfFalse(iff)); if (kit.stopped()) { @@ -1381,23 +1390,26 @@ dst_array, dst_coder, start); final_merge->init_req(3, kit.control()); final_mem->init_req(3, kit.memory(byte_adr_idx)); + final_mem_wb->init_req(3, kit.memory(byte_adr_idx_wb)); } kit.set_control(__ IfTrue(iff)); kit.set_memory(old_mem, byte_adr_idx); + kit.set_memory(old_mem_wb, byte_adr_idx_wb); if (!dcon) { // Check encoding of destination iff = kit.create_and_map_if(kit.control(), __ Bool(__ CmpI(dst_coder, __ intcon(0)), BoolTest::eq), PROB_FAIR, COUNT_UNKNOWN); old_mem = kit.memory(byte_adr_idx); + old_mem_wb = kit.memory(byte_adr_idx_wb); } if (!dcon || dbyte) { // Destination is Latin1, if (!dcon) { kit.set_control(__ IfTrue(iff)); } - getChars(kit, arg, dst_array, T_BYTE, end, final_merge, final_mem); + getChars(kit, arg, dst_array, T_BYTE, end, final_merge, final_mem, final_mem_wb); } if (!dcon || !dbyte) { // Destination is UTF16 @@ -1405,17 +1417,20 @@ if (!dcon) { kit.set_control(__ IfFalse(iff)); kit.set_memory(old_mem, byte_adr_idx); + kit.set_memory(old_mem_wb, byte_adr_idx_wb); merge_index = 3; // Account for Latin1 case } - getChars(kit, arg, dst_array, T_CHAR, end, final_merge, final_mem, merge_index); + getChars(kit, arg, dst_array, T_CHAR, end, final_merge, final_mem, final_mem_wb, merge_index); } // Final merge point for Latin1 and UTF16 case kit.set_control(final_merge); kit.set_memory(final_mem, byte_adr_idx); + kit.set_memory(final_mem_wb, byte_adr_idx_wb); C->record_for_igvn(final_merge); C->record_for_igvn(final_mem); + C->record_for_igvn(final_mem_wb); return end; } @@ -1743,6 +1758,7 @@ // Create a region for the overflow checks to merge into. int args = MAX2(sc->num_arguments(), 1); RegionNode* overflow = new RegionNode(args); + PhiNode* overflow_phi = new PhiNode(overflow, Type::MEMORY, TypePtr::BOTTOM); kit.gvn().set_type(overflow, Type::CONTROL); // Create a hook node to hold onto the individual sizes since they @@ -1784,6 +1800,9 @@ Node* p = __ Bool(__ CmpP(arg, kit.null()), BoolTest::ne); IfNode* iff = kit.create_and_map_if(kit.control(), p, PROB_MIN, COUNT_UNKNOWN); overflow->add_req(__ IfFalse(iff)); + Node* mem = __ reset_memory(); + overflow_phi->add_req(mem); + __ set_all_memory(mem); Node* notnull = __ IfTrue(iff); kit.set_control(notnull); // set control for the cast_not_null arg = kit.cast_not_null(arg, false); @@ -1900,6 +1919,9 @@ PROB_MIN, COUNT_UNKNOWN); kit.set_control(__ IfFalse(iff)); overflow->set_req(argi, __ IfTrue(iff)); + Node* mem = __ reset_memory(); + overflow_phi->set_req(argi, mem); + __ set_all_memory(mem); } } @@ -1907,6 +1929,7 @@ // Hook PreserveJVMState pjvms(&kit); kit.set_control(overflow); + kit.set_all_memory(__ gvn().transform(overflow_phi)); C->record_for_igvn(overflow); kit.uncommon_trap(Deoptimization::Reason_intrinsic, Deoptimization::Action_make_not_entrant); diff --git a/src/hotspot/share/opto/stringopts.hpp b/src/hotspot/share/opto/stringopts.hpp --- a/src/hotspot/share/opto/stringopts.hpp +++ b/src/hotspot/share/opto/stringopts.hpp @@ -42,6 +42,7 @@ // Memory slices needed for code gen int byte_adr_idx; + int byte_adr_idx_wb; // Integer.sizeTable - used for int to String conversion ciField* size_table_field; @@ -66,7 +67,7 @@ Node* int_stringSize(GraphKit& kit, Node* value); // Simplified version of Integer.getChars - void getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicType bt, Node* end, Node* final_merge, Node* final_mem, int merge_index = 0); + void getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicType bt, Node* end, Node* final_merge, Node* final_mem, Node* final_mem_wb, int merge_index = 0); // Copy the characters representing arg into dst_array starting at start Node* int_getChars(GraphKit& kit, Node* arg, Node* dst_array, Node* dst_coder, Node* start, Node* size); diff --git a/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java b/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java --- a/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java @@ -30,19 +30,9 @@ * @requires vm.flavor == "server" * @summary Stress the Shenandoah GC by trying to make old objects more likely to be garbage than young objects. * - * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=passive -XX:+ShenandoahVerify -XX:+ShenandoahDegeneratedGC TestGCBasherWithShenandoah 120000 - * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=passive -XX:+ShenandoahVerify -XX:-ShenandoahDegeneratedGC TestGCBasherWithShenandoah 120000 - * - * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot TestGCBasherWithShenandoah 120000 - * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot TestGCBasherWithShenandoah 120000 - * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=aggressive TestGCBasherWithShenandoah 120000 - * - * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=adaptive -XX:+ShenandoahVerify TestGCBasherWithShenandoah 120000 * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=traversal -XX:+ShenandoahVerify TestGCBasherWithShenandoah 120000 * - * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=adaptive TestGCBasherWithShenandoah 120000 * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=traversal TestGCBasherWithShenandoah 120000 - * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=compact TestGCBasherWithShenandoah 120000 */ public class TestGCBasherWithShenandoah { public static void main(String[] args) throws IOException {