25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "gc/shenandoah/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/shenandoahRuntime.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
40 #endif
41
42 #define __ masm->
43
44 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
45 address ShenandoahBarrierSetAssembler::_shenandoah_wb_C = NULL;
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
48 Register src, Register dst, Register count) {
49
50 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
51 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
52 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
53 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
54
55 if (type == T_OBJECT || type == T_ARRAY) {
56 #ifdef _LP64
57 if (!checkcast && !obj_int) {
58 // Save count for barrier
59 __ movptr(r11, count);
60 } else if (disjoint && obj_int) {
61 // Save dst in r11 in the disjoint case
62 __ movq(r11, dst);
63 }
64 #else
65 if (disjoint) {
937
938 __ restore_live_registers(true);
939
940 __ bind(done);
941
942 __ pop(rdx);
943 __ pop(rax);
944
945 __ epilogue();
946 }
947
948 #undef __
949
950 #endif // COMPILER1
951
952 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
953 assert(_shenandoah_wb != NULL, "need write barrier stub");
954 return _shenandoah_wb;
955 }
956
957 address ShenandoahBarrierSetAssembler::shenandoah_wb_C() {
958 assert(_shenandoah_wb_C != NULL, "need write barrier stub");
959 return _shenandoah_wb_C;
960 }
961
962 #define __ cgen->assembler()->
963
964 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test) {
965 __ align(CodeEntryAlignment);
966 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
967 address start = __ pc();
968
969 #ifdef _LP64
970 Label not_done;
971
972 // We use RDI, which also serves as argument register for slow call.
973 // RAX always holds the src object ptr, except after the slow call and
974 // the cmpxchg, then it holds the result.
975 // R8 and RCX are used as temporary registers.
976 if (!c_abi) {
977 __ push(rdi);
978 __ push(r8);
979 }
980
981 // Check for object beeing in the collection set.
982 // TODO: Can we use only 1 register here?
983 // The source object arrives here in rax.
984 // live: rax
985 // live: rdi
986 if (!c_abi) {
987 __ mov(rdi, rax);
988 } else {
989 if (rax != c_rarg0) {
990 __ mov(rax, c_rarg0);
991 }
992 }
993 if (do_cset_test) {
994 __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
995 // live: r8
996 __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
997 __ movbool(r8, Address(r8, rdi, Address::times_1));
998 // unlive: rdi
999 __ testbool(r8);
1000 // unlive: r8
1001 __ jccb(Assembler::notZero, not_done);
1002
1003 if (!c_abi) {
1004 __ pop(r8);
1005 __ pop(rdi);
1006 }
1007 __ ret(0);
1008
1009 __ bind(not_done);
1010 }
1011
1012 if (!c_abi) {
1013 __ push(rcx);
1014 }
1015
1016 if (!c_abi) {
1017 __ push(rdx);
1018 __ push(rdi);
1019 __ push(rsi);
1020 __ push(r8);
1021 __ push(r9);
1022 __ push(r10);
1023 __ push(r11);
1024 __ push(r12);
1025 __ push(r13);
1026 __ push(r14);
1027 __ push(r15);
1028 }
1029 save_vector_registers(cgen->assembler());
1030 __ movptr(rdi, rax);
1031 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi);
1032 restore_vector_registers(cgen->assembler());
1033 if (!c_abi) {
1034 __ pop(r15);
1035 __ pop(r14);
1036 __ pop(r13);
1037 __ pop(r12);
1038 __ pop(r11);
1039 __ pop(r10);
1040 __ pop(r9);
1041 __ pop(r8);
1042 __ pop(rsi);
1043 __ pop(rdi);
1044 __ pop(rdx);
1045
1046 __ pop(rcx);
1047 __ pop(r8);
1048 __ pop(rdi);
1049 }
1050 __ ret(0);
1051 #else
1052 ShouldNotReachHere();
1053 #endif
1054 return start;
1055 }
1056
1057 #undef __
1058
1059 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
1060 if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
1061 int stub_code_size = 4096;
1062 ResourceMark rm;
1063 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
1064 CodeBuffer buf(bb);
1065 StubCodeGenerator cgen(&buf);
1066 _shenandoah_wb = generate_shenandoah_wb(&cgen, false, true);
1067 _shenandoah_wb_C = generate_shenandoah_wb(&cgen, true, false);
1068 }
1069 }
|
25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "gc/shenandoah/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/shenandoahRuntime.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
40 #endif
41
42 #define __ masm->
43
44 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
45
46 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
47 Register src, Register dst, Register count) {
48
49 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
50 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
51 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
52 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
53
54 if (type == T_OBJECT || type == T_ARRAY) {
55 #ifdef _LP64
56 if (!checkcast && !obj_int) {
57 // Save count for barrier
58 __ movptr(r11, count);
59 } else if (disjoint && obj_int) {
60 // Save dst in r11 in the disjoint case
61 __ movq(r11, dst);
62 }
63 #else
64 if (disjoint) {
936
937 __ restore_live_registers(true);
938
939 __ bind(done);
940
941 __ pop(rdx);
942 __ pop(rax);
943
944 __ epilogue();
945 }
946
947 #undef __
948
949 #endif // COMPILER1
950
951 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
952 assert(_shenandoah_wb != NULL, "need write barrier stub");
953 return _shenandoah_wb;
954 }
955
956 #define __ cgen->assembler()->
957
958 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
959 __ align(CodeEntryAlignment);
960 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
961 address start = __ pc();
962
963 #ifdef _LP64
964 Label not_done;
965
966 // We use RDI, which also serves as argument register for slow call.
967 // RAX always holds the src object ptr, except after the slow call and
968 // the cmpxchg, then it holds the result.
969 // R8 and RCX are used as temporary registers.
970 __ push(rdi);
971 __ push(r8);
972
973 // Check for object beeing in the collection set.
974 // TODO: Can we use only 1 register here?
975 // The source object arrives here in rax.
976 // live: rax
977 // live: rdi
978 __ mov(rdi, rax);
979 __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
980 // live: r8
981 __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
982 __ movbool(r8, Address(r8, rdi, Address::times_1));
983 // unlive: rdi
984 __ testbool(r8);
985 // unlive: r8
986 __ jccb(Assembler::notZero, not_done);
987
988 __ pop(r8);
989 __ pop(rdi);
990 __ ret(0);
991
992 __ bind(not_done);
993
994 // Save general purpose registers
995 __ subq(rsp, 6 * wordSize);
996 // Skip rax because it's used for return anyway
997 __ movq(Address(rsp, 5 * wordSize), rcx);
998 __ movq(Address(rsp, 4 * wordSize), rdx);
999 // Skip rbx and rbp because they are callee saved in all calling conventions
1000 // skip rsp
1001 __ movq(Address(rsp, 3 * wordSize), rsi);
1002 // Skip r8 and rdi, we already pushed them above
1003 __ movq(Address(rsp, 2 * wordSize), r9);
1004 __ movq(Address(rsp, 1 * wordSize), r10);
1005 __ movq(Address(rsp, 0 * wordSize), r11);
1006 // Skip r12..r15 because they are callee saved in all calling conventions
1007
1008 save_vector_registers(cgen->assembler());
1009 __ movptr(c_rarg0, rax);
1010 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), c_rarg0);
1011 restore_vector_registers(cgen->assembler());
1012
1013 // Restore general purpose registers
1014 // Skip r12..r15
1015 __ movq(r11, Address(rsp, 0 * wordSize));
1016 __ movq(r10, Address(rsp, 1 * wordSize));
1017 __ movq(r9, Address(rsp, 2 * wordSize));
1018 // Skip r8 and rdi
1019 __ movq(rsi, Address(rsp, 3 * wordSize));
1020 // skip rdb, rsp and rbx
1021 __ movq(rdx, Address(rsp, 4 * wordSize));
1022 __ movq(rcx, Address(rsp, 5 * wordSize));
1023
1024 // Restore r8 and rdi from outermost pushes
1025 __ movq(r8, Address(rsp, 6 * wordSize));
1026 __ movq(rdi, Address(rsp, 7 * wordSize));
1027 __ addq(rsp, 8 * wordSize);
1028
1029 __ ret(0);
1030 #else
1031 ShouldNotReachHere();
1032 #endif
1033 return start;
1034 }
1035
1036 #undef __
1037
1038 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
1039 if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
1040 int stub_code_size = 4096;
1041 ResourceMark rm;
1042 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
1043 CodeBuffer buf(bb);
1044 StubCodeGenerator cgen(&buf);
1045 _shenandoah_wb = generate_shenandoah_wb(&cgen);
1046 }
1047 }
|