1 /*
2 * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "c1/c1_MacroAssembler.hpp"
26 #include "c1/c1_LIRAssembler.hpp"
27 #include "macroAssembler_x86.hpp"
28 #include "shenandoahBarrierSetAssembler_x86.hpp"
29 #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc_implementation/shenandoah/shenandoahBarrierSetC1.hpp"
31 #include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
32 #include "gc_implementation/shenandoah/shenandoahHeap.hpp"
33 #include "gc_implementation/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc_implementation/shenandoah/shenandoahRuntime.hpp"
35 #include "runtime/stubCodeGenerator.hpp"
36
37 ShenandoahBarrierSetAssembler* ShenandoahBarrierSetAssembler::bsasm() {
38 return ShenandoahBarrierSet::barrier_set()->bsasm();
39 }
40
41 #define __ masm->
42
43 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
44 assert(ShenandoahCASBarrier, "should be enabled");
45 Label is_null;
46 __ testptr(dst, dst);
47 __ jcc(Assembler::zero, is_null);
48 resolve_forward_pointer_not_null(masm, dst, tmp);
49 __ bind(is_null);
50 }
51
52 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
53 assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
54 // The below loads the mark word, checks if the lowest two bits are
55 // set, and if so, clear the lowest two bits and copy the result
56 // to dst. Otherwise it leaves dst alone.
57 // Implementing this is surprisingly awkward. I do it here by:
58 // - Inverting the mark word
59 // - Test lowest two bits == 0
60 // - If so, set the lowest two bits
61 // - Invert the result back, and copy to dst
62
63 bool borrow_reg = (tmp == noreg);
64 if (borrow_reg) {
65 // No free registers available. Make one useful.
66 tmp = LP64_ONLY(rscratch1) NOT_LP64(rdx);
67 if (tmp == dst) {
68 tmp = LP64_ONLY(rscratch2) NOT_LP64(rcx);
69 }
70 __ push(tmp);
71 }
72
73 assert_different_registers(dst, tmp);
74
75 Label done;
76 __ movptr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
77 __ notptr(tmp);
78 __ testb(tmp, markOopDesc::marked_value);
79 __ jccb(Assembler::notZero, done);
80 __ orptr(tmp, markOopDesc::marked_value);
81 __ notptr(tmp);
82 __ mov(dst, tmp);
83 __ bind(done);
84
85 if (borrow_reg) {
86 __ pop(tmp);
87 }
88 }
89
90 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
91 assert(ShenandoahLoadRefBarrier, "Should be enabled");
92
93 Label done;
94
95 #ifdef _LP64
96 Register thread = r15_thread;
97 #else
98 Register thread = rcx;
99 if (thread == dst) {
100 thread = rbx;
101 }
102 __ push(thread);
103 __ get_thread(thread);
104 #endif
105 assert_different_registers(dst, thread);
106
107 Address gc_state(thread, in_bytes(JavaThread::gc_state_offset()));
108 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
109 __ jcc(Assembler::zero, done);
110
111 {
112 __ save_vector_registers();
113
114 __ subptr(rsp, LP64_ONLY(16) NOT_LP64(8) * wordSize);
115
116 __ movptr(Address(rsp, 0 * wordSize), rax);
117 __ movptr(Address(rsp, 1 * wordSize), rcx);
118 __ movptr(Address(rsp, 2 * wordSize), rdx);
119 __ movptr(Address(rsp, 3 * wordSize), rbx);
120 // skip rsp
121 __ movptr(Address(rsp, 5 * wordSize), rbp);
122 __ movptr(Address(rsp, 6 * wordSize), rsi);
123 __ movptr(Address(rsp, 7 * wordSize), rdi);
124 #ifdef _LP64
125 __ movptr(Address(rsp, 8 * wordSize), r8);
126 __ movptr(Address(rsp, 9 * wordSize), r9);
127 __ movptr(Address(rsp, 10 * wordSize), r10);
128 __ movptr(Address(rsp, 11 * wordSize), r11);
129 __ movptr(Address(rsp, 12 * wordSize), r12);
130 __ movptr(Address(rsp, 13 * wordSize), r13);
131 __ movptr(Address(rsp, 14 * wordSize), r14);
132 __ movptr(Address(rsp, 15 * wordSize), r15);
133 #endif
134 }
135 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_interpreter), dst);
136 {
137 #ifdef _LP64
138 __ movptr(r15, Address(rsp, 15 * wordSize));
139 __ movptr(r14, Address(rsp, 14 * wordSize));
140 __ movptr(r13, Address(rsp, 13 * wordSize));
141 __ movptr(r12, Address(rsp, 12 * wordSize));
142 __ movptr(r11, Address(rsp, 11 * wordSize));
143 __ movptr(r10, Address(rsp, 10 * wordSize));
144 __ movptr(r9, Address(rsp, 9 * wordSize));
145 __ movptr(r8, Address(rsp, 8 * wordSize));
146 #endif
147 __ movptr(rdi, Address(rsp, 7 * wordSize));
148 __ movptr(rsi, Address(rsp, 6 * wordSize));
149 __ movptr(rbp, Address(rsp, 5 * wordSize));
150 // skip rsp
151 __ movptr(rbx, Address(rsp, 3 * wordSize));
152 __ movptr(rdx, Address(rsp, 2 * wordSize));
153 __ movptr(rcx, Address(rsp, 1 * wordSize));
154 if (dst != rax) {
155 __ movptr(dst, rax);
156 __ movptr(rax, Address(rsp, 0 * wordSize));
157 }
158 __ addptr(rsp, LP64_ONLY(16) NOT_LP64(8) * wordSize);
159
160 __ restore_vector_registers();
161 }
162 __ bind(done);
163
164 #ifndef _LP64
165 __ pop(thread);
166 #endif
167 }
168
169 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
170 if (ShenandoahStoreValEnqueueBarrier) {
171 storeval_barrier_impl(masm, dst, tmp);
172 }
173 }
174
175 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
176 assert(ShenandoahStoreValEnqueueBarrier, "should be enabled");
177
178 if (dst == noreg) return;
179
180 if (ShenandoahStoreValEnqueueBarrier) {
181 // The set of registers to be saved+restored is the same as in the write-barrier above.
182 // Those are the commonly used registers in the interpreter.
183 __ pusha();
184 // __ push_callee_saved_registers();
185 __ subptr(rsp, 2 * Interpreter::stackElementSize);
186 __ movdbl(Address(rsp, 0), xmm0);
187
188 #ifdef _LP64
189 Register thread = r15_thread;
190 #else
191 Register thread = rcx;
192 if (thread == dst || thread == tmp) {
193 thread = rdi;
194 }
195 if (thread == dst || thread == tmp) {
196 thread = rbx;
197 }
198 __ get_thread(thread);
199 #endif
200 assert_different_registers(dst, tmp, thread);
201
202 __ g1_write_barrier_pre(noreg, dst, thread, tmp, true, false);
203 __ movdbl(xmm0, Address(rsp, 0));
204 __ addptr(rsp, 2 * Interpreter::stackElementSize);
205 //__ pop_callee_saved_registers();
206 __ popa();
207 }
208 }
209
210 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) {
211 if (ShenandoahLoadRefBarrier) {
212 Label done;
213 __ testptr(dst, dst);
214 __ jcc(Assembler::zero, done);
215 load_reference_barrier_not_null(masm, dst);
216 __ bind(done);
217 }
218 }
219
220 // Special Shenandoah CAS implementation that handles false negatives
221 // due to concurrent evacuation.
222 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
223 Register res, Address addr, Register oldval, Register newval,
224 bool exchange, Register tmp1, Register tmp2) {
225 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
226 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
227
228 Label retry, done;
229
230 // Remember oldval for retry logic below
231 #ifdef _LP64
232 if (UseCompressedOops) {
233 __ movl(tmp1, oldval);
234 } else
235 #endif
236 {
237 __ movptr(tmp1, oldval);
238 }
239
240 // Step 1. Try to CAS with given arguments. If successful, then we are done,
241 // and can safely return.
242 if (os::is_MP()) __ lock();
243 #ifdef _LP64
244 if (UseCompressedOops) {
245 __ cmpxchgl(newval, addr);
246 } else
247 #endif
248 {
249 __ cmpxchgptr(newval, addr);
250 }
251 __ jcc(Assembler::equal, done, true);
252
253 // Step 2. CAS had failed. This may be a false negative.
254 //
255 // The trouble comes when we compare the to-space pointer with the from-space
256 // pointer to the same object. To resolve this, it will suffice to resolve both
257 // oldval and the value from memory -- this will give both to-space pointers.
258 // If they mismatch, then it was a legitimate failure.
259 //
260 #ifdef _LP64
261 if (UseCompressedOops) {
262 __ decode_heap_oop(tmp1);
263 }
264 #endif
265 resolve_forward_pointer(masm, tmp1);
266
267 #ifdef _LP64
268 if (UseCompressedOops) {
269 __ movl(tmp2, oldval);
270 __ decode_heap_oop(tmp2);
271 } else
272 #endif
273 {
274 __ movptr(tmp2, oldval);
275 }
276 resolve_forward_pointer(masm, tmp2);
277
278 __ cmpptr(tmp1, tmp2);
279 __ jcc(Assembler::notEqual, done, true);
280
281 // Step 3. Try to CAS again with resolved to-space pointers.
282 //
283 // Corner case: it may happen that somebody stored the from-space pointer
284 // to memory while we were preparing for retry. Therefore, we can fail again
285 // on retry, and so need to do this in loop, always resolving the failure
286 // witness.
287 __ bind(retry);
288 if (os::is_MP()) __ lock();
289 #ifdef _LP64
290 if (UseCompressedOops) {
291 __ cmpxchgl(newval, addr);
292 } else
293 #endif
294 {
295 __ cmpxchgptr(newval, addr);
296 }
297 __ jcc(Assembler::equal, done, true);
298
299 #ifdef _LP64
300 if (UseCompressedOops) {
301 __ movl(tmp2, oldval);
302 __ decode_heap_oop(tmp2);
303 } else
304 #endif
305 {
306 __ movptr(tmp2, oldval);
307 }
308 resolve_forward_pointer(masm, tmp2);
309
310 __ cmpptr(tmp1, tmp2);
311 __ jcc(Assembler::equal, retry, true);
312
313 // Step 4. If we need a boolean result out of CAS, check the flag again,
314 // and promote the result. Note that we handle the flag from both the CAS
315 // itself and from the retry loop.
316 __ bind(done);
317 if (!exchange) {
318 assert(res != NULL, "need result register");
319 #ifdef _LP64
320 __ setb(Assembler::equal, res);
321 __ movzbl(res, res);
322 #else
323 // Need something else to clean the result, because some registers
324 // do not have byte encoding that movzbl wants. Cannot do the xor first,
325 // because it modifies the flags.
326 Label res_non_zero;
327 __ movptr(res, 1);
328 __ jcc(Assembler::equal, res_non_zero, true);
329 __ xorptr(res, res);
330 __ bind(res_non_zero);
331 #endif
332 }
333 }
334
335 #undef __
336
337 #ifdef COMPILER1
338
339 #define __ ce->masm()->
340
341 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
342 __ bind(*stub->entry());
343
344 Label done;
345 Register obj = stub->obj()->as_register();
346 Register res = stub->result()->as_register();
347
348 if (res != obj) {
349 __ mov(res, obj);
350 }
351
352 // Check for null.
353 if (stub->needs_null_check()) {
354 __ testptr(res, res);
355 __ jcc(Assembler::zero, done);
356 }
357
358 load_reference_barrier_not_null(ce->masm(), res);
359
360 __ bind(done);
361 __ jmp(*stub->continuation());
362 }
363
364 #undef __
365
366 #endif // COMPILER1