30 #include "gc/shenandoah/shenandoahRuntime.hpp"
31 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "interpreter/interp_masm.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/thread.hpp"
36 #include "utilities/macros.hpp"
37 #ifdef COMPILER1
38 #include "c1/c1_LIRAssembler.hpp"
39 #include "c1/c1_MacroAssembler.hpp"
40 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
41 #endif
42
43 #define __ masm->
44
45 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
48 Register src, Register dst, Register count) {
49
50 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
51 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
52 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
53 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
54
55 if (type == T_OBJECT || type == T_ARRAY) {
56 #ifdef _LP64
57 if (!checkcast) {
58 if (!obj_int) {
59 // Save count for barrier
60 __ movptr(r11, count);
61 } else if (disjoint) {
62 // Save dst in r11 in the disjoint case
63 __ movq(r11, dst);
64 }
65 }
66 #else
67 if (disjoint) {
68 __ mov(rdx, dst); // save 'to'
69 }
70 #endif
71
72 if (ShenandoahSATBBarrier && !dest_uninitialized) {
73 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
74 assert_different_registers(dst, count, thread); // we don't care about src here?
75 #ifndef _LP64
76 __ push(thread);
77 __ get_thread(thread);
78 #endif
79
80 Label done;
81 // Short-circuit if count == 0.
82 __ testptr(count, count);
83 __ jcc(Assembler::zero, done);
84
85 // Avoid runtime call when not marking.
86 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
87 __ testb(gc_state, ShenandoahHeap::MARKING);
88 __ jcc(Assembler::zero, done);
89
90 __ pusha(); // push registers
91 #ifdef _LP64
92 if (count == c_rarg0) {
93 if (dst == c_rarg1) {
94 // exactly backwards!!
95 __ xchgptr(c_rarg1, c_rarg0);
96 } else {
97 __ movptr(c_rarg1, count);
98 __ movptr(c_rarg0, dst);
99 }
100 } else {
101 __ movptr(c_rarg0, dst);
102 __ movptr(c_rarg1, count);
103 }
104 if (UseCompressedOops) {
105 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
106 } else {
107 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
108 }
109 #else
110 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
111 dst, count);
112 #endif
113 __ popa();
114 __ bind(done);
115 NOT_LP64(__ pop(thread);)
116 }
117 }
118
119 }
120
121 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
122 Register src, Register dst, Register count) {
123 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
124 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
125 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
126 Register tmp = rax;
127
128 if (type == T_OBJECT || type == T_ARRAY) {
129 #ifdef _LP64
130 if (!checkcast) {
131 if (!obj_int) {
132 // Save count for barrier
133 count = r11;
134 } else if (disjoint && obj_int) {
135 // Use the saved dst in the disjoint case
136 dst = r11;
137 }
138 } else {
139 tmp = rscratch1;
140 }
141 #else
142 if (disjoint) {
143 __ mov(dst, rdx); // restore 'to'
144 }
145 #endif
146
147 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
148 assert_different_registers(dst, thread); // do we care about src at all here?
149
150 #ifndef _LP64
151 __ push(thread);
152 __ get_thread(thread);
153 #endif
154
155 // Short-circuit if count == 0.
156 Label done;
157 __ testptr(count, count);
158 __ jcc(Assembler::zero, done);
159
160 // Skip runtime call if no forwarded objects.
161 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
162 __ testb(gc_state, ShenandoahHeap::UPDATEREFS);
163 __ jcc(Assembler::zero, done);
164
165 __ pusha(); // push registers (overkill)
166 #ifdef _LP64
167 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
168 assert_different_registers(c_rarg1, dst);
169 __ mov(c_rarg1, count);
170 __ mov(c_rarg0, dst);
171 } else {
172 assert_different_registers(c_rarg0, count);
173 __ mov(c_rarg0, dst);
174 __ mov(c_rarg1, count);
175 }
176 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
177 #else
178 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry),
179 dst, count);
180 #endif
181 __ popa();
182
183 __ bind(done);
184 NOT_LP64(__ pop(thread);)
185 }
186 }
187
188 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
189 Register obj,
190 Register pre_val,
191 Register thread,
192 Register tmp,
193 bool tosca_live,
194 bool expand_call) {
195
196 if (ShenandoahSATBBarrier) {
197 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
198 }
199 }
200
201 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
202 Register obj,
203 Register pre_val,
204 Register thread,
205 Register tmp,
|
30 #include "gc/shenandoah/shenandoahRuntime.hpp"
31 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "interpreter/interp_masm.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/thread.hpp"
36 #include "utilities/macros.hpp"
37 #ifdef COMPILER1
38 #include "c1/c1_LIRAssembler.hpp"
39 #include "c1/c1_MacroAssembler.hpp"
40 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
41 #endif
42
43 #define __ masm->
44
45 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
48 Register src, Register dst, Register count) {
49
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51
52 if (type == T_OBJECT || type == T_ARRAY) {
53
54 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
55 #ifdef _LP64
56 Register thread = r15_thread;
57 #else
58 Register thread = rax;
59 if (thread == src || thread == dst || thread == count) {
60 thread = rbx;
61 }
62 if (thread == src || thread == dst || thread == count) {
63 thread = rcx;
64 }
65 if (thread == src || thread == dst || thread == count) {
66 thread = rdx;
67 }
68 __ push(thread);
69 __ get_thread(thread);
70 #endif
71 assert_different_registers(src, dst, count, thread);
72
73 Label done;
74 // Short-circuit if count == 0.
75 __ testptr(count, count);
76 __ jcc(Assembler::zero, done);
77
78 // Avoid runtime call when not marking.
79 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
80 int flags = ShenandoahHeap::HAS_FORWARDED;
81 if (!dest_uninitialized) {
82 flags |= ShenandoahHeap::MARKING;
83 }
84 __ testb(gc_state, flags);
85 __ jcc(Assembler::zero, done);
86
87 __ pusha(); // push registers
88 #ifdef _LP64
89 assert(src == rdi, "expected");
90 assert(dst == rsi, "expected");
91 assert(count == rdx, "expected");
92 if (UseCompressedOops) {
93 if (dest_uninitialized) {
94 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
95 } else {
96 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
97 }
98 } else {
99 if (dest_uninitialized) {
100 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
101 } else {
102 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
103 }
104 }
105 #else
106 if (dest_uninitialized) {
107 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry),
108 src, dst, count);
109 } else {
110 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
111 src, dst, count);
112 }
113 #endif
114 __ popa();
115 __ bind(done);
116 NOT_LP64(__ pop(thread);)
117 }
118 }
119
120 }
121
122 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
123 Register obj,
124 Register pre_val,
125 Register thread,
126 Register tmp,
127 bool tosca_live,
128 bool expand_call) {
129
130 if (ShenandoahSATBBarrier) {
131 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
132 }
133 }
134
135 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
136 Register obj,
137 Register pre_val,
138 Register thread,
139 Register tmp,
|