< prev index next >
src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
Print this page
*** 21,31 ****
*
*/
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
- #include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahHeuristics.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
--- 21,30 ----
*** 311,348 ****
if(tosca_live) __ pop(rax);
__ bind(done);
}
! void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
assert(ShenandoahCASBarrier, "should be enabled");
Label is_null;
__ testptr(dst, dst);
__ jcc(Assembler::zero, is_null);
! resolve_forward_pointer_not_null(masm, dst);
__ bind(is_null);
}
! void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
! __ movptr(dst, Address(dst, ShenandoahForwarding::byte_offset()));
}
void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");
#ifdef _LP64
Label done;
Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
! __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
! __ jccb(Assembler::zero, done);
!
! // Heap is unstable, need to perform the resolve even if LRB is inactive
! resolve_forward_pointer_not_null(masm, dst);
!
! __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
__ jccb(Assembler::zero, done);
if (dst != rax) {
__ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
}
--- 310,357 ----
if(tosca_live) __ pop(rax);
__ bind(done);
}
! void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
assert(ShenandoahCASBarrier, "should be enabled");
Label is_null;
__ testptr(dst, dst);
__ jcc(Assembler::zero, is_null);
! resolve_forward_pointer_not_null(masm, dst, tmp);
__ bind(is_null);
}
! void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
! // The below loads the mark word, checks if the lowest two bits are
! // set, and if so, clear the lowest two bits and copy the result
! // to dst. Otherwise it leaves dst alone.
! // Implementing this is surprisingly awkward. I do it here by:
! // - Inverting the mark word
! // - Test lowest two bits == 0
! // - If so, set the lowest two bits
! // - Invert the result back, and copy to dst
! Label done;
! __ movptr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
! __ notptr(tmp);
! __ testb(tmp, markOopDesc::marked_value);
! __ jccb(Assembler::notZero, done);
! __ orptr(tmp, markOopDesc::marked_value);
! __ notptr(tmp);
! __ mov(dst, tmp);
! __ bind(done);
}
void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");
#ifdef _LP64
Label done;
Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
! __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
__ jccb(Assembler::zero, done);
if (dst != rax) {
__ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
}
*** 473,531 ****
} else {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
}
}
- void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
- Register thread, Register obj,
- Register var_size_in_bytes,
- int con_size_in_bytes,
- Register t1, Register t2,
- Label& slow_case) {
- assert_different_registers(obj, t1, t2);
- assert_different_registers(obj, var_size_in_bytes, t1);
- Register end = t2;
- if (!thread->is_valid()) {
- #ifdef _LP64
- thread = r15_thread;
- #else
- assert(t1->is_valid(), "need temp reg");
- thread = t1;
- __ get_thread(thread);
- #endif
- }
-
- __ verify_tlab();
-
- __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
- if (var_size_in_bytes == noreg) {
- __ lea(end, Address(obj, con_size_in_bytes + ShenandoahForwarding::byte_size()));
- } else {
- __ addptr(var_size_in_bytes, ShenandoahForwarding::byte_size());
- __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
- }
- __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
- __ jcc(Assembler::above, slow_case);
-
- // update the tlab top pointer
- __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
-
- // Initialize brooks pointer
- #ifdef _LP64
- __ incrementq(obj, ShenandoahForwarding::byte_size());
- #else
- __ incrementl(obj, ShenandoahForwarding::byte_size());
- #endif
- __ movptr(Address(obj, ShenandoahForwarding::byte_offset()), obj);
-
- // recover var_size_in_bytes if necessary
- if (var_size_in_bytes == end) {
- __ subptr(var_size_in_bytes, obj);
- }
- __ verify_tlab();
- }
-
// Special Shenandoah CAS implementation that handles false negatives
// due to concurrent evacuation.
#ifndef _LP64
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
Register res, Address addr, Register oldval, Register newval,
--- 482,491 ----
*** 534,544 ****
Unimplemented();
}
#else
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
Register res, Address addr, Register oldval, Register newval,
! bool exchange, Register tmp1, Register tmp2) {
assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
Label retry, done;
--- 494,504 ----
Unimplemented();
}
#else
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
Register res, Address addr, Register oldval, Register newval,
! bool exchange, Register tmp1, Register tmp2, Register tmp3) {
assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
Label retry, done;
*** 567,585 ****
// If they mismatch, then it was a legitimate failure.
//
if (UseCompressedOops) {
__ decode_heap_oop(tmp1);
}
! resolve_forward_pointer(masm, tmp1);
if (UseCompressedOops) {
__ movl(tmp2, oldval);
__ decode_heap_oop(tmp2);
} else {
__ movptr(tmp2, oldval);
}
! resolve_forward_pointer(masm, tmp2);
__ cmpptr(tmp1, tmp2);
__ jcc(Assembler::notEqual, done, true);
// Step 3. Try to CAS again with resolved to-space pointers.
--- 527,545 ----
// If they mismatch, then it was a legitimate failure.
//
if (UseCompressedOops) {
__ decode_heap_oop(tmp1);
}
! resolve_forward_pointer(masm, tmp1, tmp3);
if (UseCompressedOops) {
__ movl(tmp2, oldval);
__ decode_heap_oop(tmp2);
} else {
__ movptr(tmp2, oldval);
}
! resolve_forward_pointer(masm, tmp2, tmp3);
__ cmpptr(tmp1, tmp2);
__ jcc(Assembler::notEqual, done, true);
// Step 3. Try to CAS again with resolved to-space pointers.
*** 601,611 ****
__ movl(tmp2, oldval);
__ decode_heap_oop(tmp2);
} else {
__ movptr(tmp2, oldval);
}
! resolve_forward_pointer(masm, tmp2);
__ cmpptr(tmp1, tmp2);
__ jcc(Assembler::equal, retry, true);
// Step 4. If we need a boolean result out of CAS, check the flag again,
--- 561,571 ----
__ movl(tmp2, oldval);
__ decode_heap_oop(tmp2);
} else {
__ movptr(tmp2, oldval);
}
! resolve_forward_pointer(masm, tmp2, tmp3);
__ cmpptr(tmp1, tmp2);
__ jcc(Assembler::equal, retry, true);
// Step 4. If we need a boolean result out of CAS, check the flag again,
*** 849,859 ****
__ align(CodeEntryAlignment);
StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
address start = __ pc();
#ifdef _LP64
! Label not_done;
// We use RDI, which also serves as argument register for slow call.
// RAX always holds the src object ptr, except after the slow call and
// the cmpxchg, then it holds the result.
// R8 and RCX are used as temporary registers.
--- 809,819 ----
__ align(CodeEntryAlignment);
StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
address start = __ pc();
#ifdef _LP64
! Label resolve_oop, slow_path;
// We use RDI, which also serves as argument register for slow call.
// RAX always holds the src object ptr, except after the slow call and
// the cmpxchg, then it holds the result.
// R8 and RCX are used as temporary registers.
*** 871,887 ****
__ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
__ movbool(r8, Address(r8, rdi, Address::times_1));
// unlive: rdi
__ testbool(r8);
// unlive: r8
! __ jccb(Assembler::notZero, not_done);
__ pop(r8);
__ pop(rdi);
__ ret(0);
! __ bind(not_done);
__ push(rcx);
__ push(rdx);
__ push(rdi);
__ push(rsi);
--- 831,865 ----
__ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
__ movbool(r8, Address(r8, rdi, Address::times_1));
// unlive: rdi
__ testbool(r8);
// unlive: r8
! __ jccb(Assembler::notZero, resolve_oop);
!
! __ pop(r8);
! __ pop(rdi);
! __ ret(0);
!
! __ bind(resolve_oop);
!
! __ movptr(r8, Address(rax, oopDesc::mark_offset_in_bytes()));
! // Test if both lowest bits are set. We trick it by negating the bits
! // then test for both bits clear.
! __ notptr(r8);
! __ testb(r8, markOopDesc::marked_value);
! __ jccb(Assembler::notZero, slow_path);
! // Clear both lower bits. It's still inverted, so set them, and then invert back.
! __ orptr(r8, markOopDesc::marked_value);
! __ notptr(r8);
! // At this point, r8 contains the decoded forwarding pointer.
! __ mov(rax, r8);
__ pop(r8);
__ pop(rdi);
__ ret(0);
! __ bind(slow_path);
__ push(rcx);
__ push(rdx);
__ push(rdi);
__ push(rsi);
< prev index next >