1 /*
2 * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "c1/c1_MacroAssembler.hpp"
26 #include "c1/c1_LIRAssembler.hpp"
27 #include "macroAssembler_x86.hpp"
28 #include "shenandoahBarrierSetAssembler_x86.hpp"
29 #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc_implementation/shenandoah/shenandoahBarrierSetC1.hpp"
31 #include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
32 #include "gc_implementation/shenandoah/shenandoahHeap.hpp"
33 #include "gc_implementation/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc_implementation/shenandoah/shenandoahRuntime.hpp"
35 #include "runtime/stubCodeGenerator.hpp"
36
37 ShenandoahBarrierSetAssembler* ShenandoahBarrierSetAssembler::bsasm() {
38 return ShenandoahBarrierSet::barrier_set()->bsasm();
39 }
40
41 #define __ masm->
42
43 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
44 assert(ShenandoahCASBarrier, "should be enabled");
45 Label is_null;
46 __ testptr(dst, dst);
47 __ jcc(Assembler::zero, is_null);
48 resolve_forward_pointer_not_null(masm, dst, tmp);
49 __ bind(is_null);
50 }
51
52 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
53 assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
54 // The below loads the mark word, checks if the lowest two bits are
55 // set, and if so, clear the lowest two bits and copy the result
56 // to dst. Otherwise it leaves dst alone.
57 // Implementing this is surprisingly awkward. I do it here by:
58 // - Inverting the mark word
59 // - Test lowest two bits == 0
60 // - If so, set the lowest two bits
61 // - Invert the result back, and copy to dst
62
63 bool borrow_reg = (tmp == noreg);
64 if (borrow_reg) {
65 // No free registers available. Make one useful.
66 tmp = rscratch1;
67 __ push(tmp);
68 }
69
70 Label done;
71 __ movptr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
72 __ notptr(tmp);
73 __ testb(tmp, markOopDesc::marked_value);
74 __ jccb(Assembler::notZero, done);
75 __ orptr(tmp, markOopDesc::marked_value);
76 __ notptr(tmp);
77 __ mov(dst, tmp);
78 __ bind(done);
79
80 if (borrow_reg) {
81 __ pop(tmp);
82 }
83 }
84
85 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
86 assert(ShenandoahLoadRefBarrier, "Should be enabled");
87 #ifdef _LP64
88 Label done;
89
90 Address gc_state(r15_thread, in_bytes(JavaThread::gc_state_offset()));
91 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
92 __ jcc(Assembler::zero, done);
93
94 {
95 __ save_vector_registers();
96
97 __ subq(rsp, 16 * wordSize);
98
99 __ movq(Address(rsp, 15 * wordSize), rax);
100 __ movq(Address(rsp, 14 * wordSize), rcx);
101 __ movq(Address(rsp, 13 * wordSize), rdx);
102 __ movq(Address(rsp, 12 * wordSize), rbx);
103 // skip rsp
104 __ movq(Address(rsp, 10 * wordSize), rbp);
105 __ movq(Address(rsp, 9 * wordSize), rsi);
106 __ movq(Address(rsp, 8 * wordSize), rdi);
107 __ movq(Address(rsp, 7 * wordSize), r8);
108 __ movq(Address(rsp, 6 * wordSize), r9);
109 __ movq(Address(rsp, 5 * wordSize), r10);
110 __ movq(Address(rsp, 4 * wordSize), r11);
111 __ movq(Address(rsp, 3 * wordSize), r12);
112 __ movq(Address(rsp, 2 * wordSize), r13);
113 __ movq(Address(rsp, wordSize), r14);
114 __ movq(Address(rsp, 0), r15);
115 }
116 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_interpreter), dst);
117 {
118 __ movq(r15, Address(rsp, 0));
119 __ movq(r14, Address(rsp, wordSize));
120 __ movq(r13, Address(rsp, 2 * wordSize));
121 __ movq(r12, Address(rsp, 3 * wordSize));
122 __ movq(r11, Address(rsp, 4 * wordSize));
123 __ movq(r10, Address(rsp, 5 * wordSize));
124 __ movq(r9, Address(rsp, 6 * wordSize));
125 __ movq(r8, Address(rsp, 7 * wordSize));
126 __ movq(rdi, Address(rsp, 8 * wordSize));
127 __ movq(rsi, Address(rsp, 9 * wordSize));
128 __ movq(rbp, Address(rsp, 10 * wordSize));
129 // skip rsp
130 __ movq(rbx, Address(rsp, 12 * wordSize));
131 __ movq(rdx, Address(rsp, 13 * wordSize));
132 __ movq(rcx, Address(rsp, 14 * wordSize));
133 if (dst != rax) {
134 __ movq(dst, rax);
135 __ movq(rax, Address(rsp, 15 * wordSize));
136 }
137 __ addq(rsp, 16 * wordSize);
138
139 __ restore_vector_registers();
140 }
141 __ bind(done);
142 #else
143 Unimplemented();
144 #endif
145 }
146
147 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) {
148 if (ShenandoahLoadRefBarrier) {
149 Label done;
150 __ testptr(dst, dst);
151 __ jcc(Assembler::zero, done);
152 load_reference_barrier_not_null(masm, dst);
153 __ bind(done);
154 }
155 }
156
157 // Special Shenandoah CAS implementation that handles false negatives
158 // due to concurrent evacuation.
159 #ifndef _LP64
160 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
161 Register res, Address addr, Register oldval, Register newval,
162 bool exchange, Register tmp1, Register tmp2) {
163 // Shenandoah has no 32-bit version for this.
164 Unimplemented();
165 }
166 #else
167 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
168 Register res, Address addr, Register oldval, Register newval,
169 bool exchange, Register tmp1, Register tmp2) {
170 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
171 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
172
173 Label retry, done;
174
175 // Remember oldval for retry logic below
176 if (UseCompressedOops) {
177 __ movl(tmp1, oldval);
178 } else {
179 __ movptr(tmp1, oldval);
180 }
181
182 // Step 1. Try to CAS with given arguments. If successful, then we are done,
183 // and can safely return.
184 if (os::is_MP()) __ lock();
185 if (UseCompressedOops) {
186 __ cmpxchgl(newval, addr);
187 } else {
188 __ cmpxchgptr(newval, addr);
189 }
190 __ jcc(Assembler::equal, done, true);
191
192 // Step 2. CAS had failed. This may be a false negative.
193 //
194 // The trouble comes when we compare the to-space pointer with the from-space
195 // pointer to the same object. To resolve this, it will suffice to resolve both
196 // oldval and the value from memory -- this will give both to-space pointers.
197 // If they mismatch, then it was a legitimate failure.
198 //
199 if (UseCompressedOops) {
200 __ decode_heap_oop(tmp1);
201 }
202 resolve_forward_pointer(masm, tmp1);
203
204 if (UseCompressedOops) {
205 __ movl(tmp2, oldval);
206 __ decode_heap_oop(tmp2);
207 } else {
208 __ movptr(tmp2, oldval);
209 }
210 resolve_forward_pointer(masm, tmp2);
211
212 __ cmpptr(tmp1, tmp2);
213 __ jcc(Assembler::notEqual, done, true);
214
215 // Step 3. Try to CAS again with resolved to-space pointers.
216 //
217 // Corner case: it may happen that somebody stored the from-space pointer
218 // to memory while we were preparing for retry. Therefore, we can fail again
219 // on retry, and so need to do this in loop, always resolving the failure
220 // witness.
221 __ bind(retry);
222 if (os::is_MP()) __ lock();
223 if (UseCompressedOops) {
224 __ cmpxchgl(newval, addr);
225 } else {
226 __ cmpxchgptr(newval, addr);
227 }
228 __ jcc(Assembler::equal, done, true);
229
230 if (UseCompressedOops) {
231 __ movl(tmp2, oldval);
232 __ decode_heap_oop(tmp2);
233 } else {
234 __ movptr(tmp2, oldval);
235 }
236 resolve_forward_pointer(masm, tmp2);
237
238 __ cmpptr(tmp1, tmp2);
239 __ jcc(Assembler::equal, retry, true);
240
241 // Step 4. If we need a boolean result out of CAS, check the flag again,
242 // and promote the result. Note that we handle the flag from both the CAS
243 // itself and from the retry loop.
244 __ bind(done);
245 if (!exchange) {
246 assert(res != NULL, "need result register");
247 __ setb(Assembler::equal, res);
248 __ movzbl(res, res);
249 }
250 }
251 #endif // LP64
252
253 #undef __
254
255 #ifdef COMPILER1
256
257 #define __ ce->masm()->
258
259 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
260 __ bind(*stub->entry());
261
262 Label done;
263 Register obj = stub->obj()->as_register();
264 Register res = stub->result()->as_register();
265
266 if (res != obj) {
267 __ mov(res, obj);
268 }
269
270 // Check for null.
271 if (stub->needs_null_check()) {
272 __ testptr(res, res);
273 __ jcc(Assembler::zero, done);
274 }
275
276 load_reference_barrier_not_null(ce->masm(), res);
277
278 __ bind(done);
279 __ jmp(*stub->continuation());
280 }
281
282 #undef __
283
284 #endif // COMPILER1