1 /*
2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "asm/macroAssembler.inline.hpp"
26 #include "code/codeBlob.hpp"
27 #include "gc/z/zBarrier.inline.hpp"
28 #include "gc/z/zBarrierSet.hpp"
29 #include "gc/z/zBarrierSetAssembler.hpp"
30 #include "gc/z/zBarrierSetRuntime.hpp"
31 #include "memory/resourceArea.hpp"
32 #ifdef COMPILER1
33 #include "c1/c1_LIRAssembler.hpp"
34 #include "c1/c1_MacroAssembler.hpp"
35 #include "gc/z/c1/zBarrierSetC1.hpp"
36 #endif // COMPILER1
37
38 #include "gc/z/zThreadLocalData.hpp"
39
40 ZBarrierSetAssembler::ZBarrierSetAssembler() :
41 _load_barrier_slow_stub(),
42 _load_barrier_weak_slow_stub() {}
43
44 #ifdef PRODUCT
45 #define BLOCK_COMMENT(str) /* nothing */
46 #else
47 #define BLOCK_COMMENT(str) __ block_comment(str)
48 #endif
49
50 #undef __
51 #define __ masm->
52
53 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
54 DecoratorSet decorators,
55 BasicType type,
56 Register dst,
57 Address src,
58 Register tmp1,
59 Register tmp_thread) {
60 if (!ZBarrierSet::barrier_needed(decorators, type)) {
61 // Barrier not needed
62 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
63 return;
64 }
65
66 // rscratch1 can be passed as src or dst, so don't use it.
67 RegSet savedRegs = RegSet::of(rscratch2, rheapbase);
68
69 Label done;
70 assert_different_registers(rheapbase, rscratch2, dst);
71 assert_different_registers(rheapbase, rscratch2, src.base());
72
73 __ push(savedRegs, sp);
74
75 // Load bad mask into scratch register.
76 __ ldr(rheapbase, address_bad_mask_from_thread(rthread));
77 __ lea(rscratch2, src);
78 __ ldr(dst, src);
79
80 // Test reference against bad mask. If mask bad, then we need to fix it up.
81 __ tst(dst, rheapbase);
82 __ br(Assembler::EQ, done);
83
84 __ enter();
85
86 __ push(RegSet::range(r0,r28) - RegSet::of(dst), sp);
87
88 if (c_rarg0 != dst) {
89 __ mov(c_rarg0, dst);
90 }
91 __ mov(c_rarg1, rscratch2);
92
93 int step = 4 * wordSize;
94 __ mov(rscratch1, -step);
95 __ sub(sp, sp, step);
96
97 for (int i = 28; i >= 4; i -= 4) {
98 __ st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
99 as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch1)));
100 }
101
102 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
103
104 for (int i = 0; i <= 28; i += 4) {
105 __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
106 as_FloatRegister(i+3), __ T1D, Address(__ post(sp, step)));
107 }
108
109 // Make sure dst has the return value.
110 if (dst != r0) {
111 __ mov(dst, r0);
112 }
113
114 __ pop(RegSet::range(r0,r28) - RegSet::of(dst), sp);
115 __ leave();
116
117 __ bind(done);
118
119 // Restore tmps
120 __ pop(savedRegs, sp);
121 }
122
123 #ifdef ASSERT
124
125 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
126 DecoratorSet decorators,
127 BasicType type,
128 Address dst,
129 Register val,
130 Register tmp1,
131 Register tmp2) {
132 // Verify value
133 if (type == T_OBJECT || type == T_ARRAY) {
134 // Note that src could be noreg, which means we
135 // are storing null and can skip verification.
136 if (val != noreg) {
137 Label done;
138
139 // tmp1 and tmp2 are often set to noreg.
140 RegSet savedRegs = RegSet::of(rscratch1);
141 __ push(savedRegs, sp);
142
143 __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
144 __ tst(val, rscratch1);
145 __ br(Assembler::EQ, done);
146 __ stop("Verify oop store failed");
147 __ should_not_reach_here();
148 __ bind(done);
149 __ pop(savedRegs, sp);
150 }
151 }
152
153 // Store value
154 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
155 }
156
157 #endif // ASSERT
158
159 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
160 DecoratorSet decorators,
161 bool is_oop,
162 Register src,
163 Register dst,
164 Register count,
165 RegSet saved_regs) {
166 if (!is_oop) {
167 // Barrier not needed
168 return;
169 }
170
171 BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
172
173 assert_different_registers(src, count, rscratch1);
174
175 __ pusha();
176
177 if (count == c_rarg0) {
178 if (src == c_rarg1) {
179 // exactly backwards!!
180 __ mov(rscratch1, c_rarg0);
181 __ mov(c_rarg0, c_rarg1);
182 __ mov(c_rarg1, rscratch1);
183 } else {
184 __ mov(c_rarg1, count);
185 __ mov(c_rarg0, src);
186 }
187 } else {
188 __ mov(c_rarg0, src);
189 __ mov(c_rarg1, count);
190 }
191
192 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
193
194 __ popa();
195 BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
196 }
197
198 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
199 Register jni_env,
200 Register robj,
201 Register tmp,
202 Label& slowpath) {
203 BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
204
205 assert_different_registers(jni_env, robj, tmp);
206
207 // Resolve jobject
208 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
209
210 // The Address offset is too large to direct load - -784. Our range is +127, -128.
211 __ mov(tmp, (long int)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) -
212 in_bytes(JavaThread::jni_environment_offset())));
213 // Load address bad mask
214 __ add(tmp, jni_env, tmp);
215 __ ldr(tmp, Address(tmp));
216
217 // Check address bad mask
218 __ tst(robj, tmp);
219 __ br(Assembler::NE, slowpath);
220
221 BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
222 }
223
224 #ifdef COMPILER1
225
226 #undef __
227 #define __ ce->masm()->
228
229 void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
230 LIR_Opr ref) const {
231 assert_different_registers(rheapbase, rthread, ref->as_register());
232
233 __ ldr(rheapbase, address_bad_mask_from_thread(rthread));
234 __ tst(ref->as_register(), rheapbase);
235 }
236
237 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
238 ZLoadBarrierStubC1* stub) const {
239 // Stub entry
240 __ bind(*stub->entry());
241
242 Register ref = stub->ref()->as_register();
243 Register ref_addr = noreg;
244 Register tmp = noreg;
245
246 if (stub->tmp()->is_valid()) {
247 // Load address into tmp register
248 ce->leal(stub->ref_addr(), stub->tmp());
249 ref_addr = tmp = stub->tmp()->as_pointer_register();
250 } else {
251 // Address already in register
252 ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
253 }
254
255 assert_different_registers(ref, ref_addr, noreg);
256
257 // Save r0 unless it is the result or tmp register
258 // Set up SP to accomodate parameters and maybe r0..
259 if (ref != r0 && tmp != r0) {
260 __ sub(sp, sp, 32);
261 __ str(r0, Address(sp, 16));
262 } else {
263 __ sub(sp, sp, 16);
264 }
265
266 // Setup arguments and call runtime stub
267 ce->store_parameter(ref_addr, 1);
268 ce->store_parameter(ref, 0);
269
270 __ far_call(stub->runtime_stub());
271
272 // Verify result
273 __ verify_oop(r0, "Bad oop");
274
275 // Move result into place
276 if (ref != r0) {
277 __ mov(ref, r0);
278 }
279
280 // Restore r0 unless it is the result or tmp register
281 if (ref != r0 && tmp != r0) {
282 __ ldr(r0, Address(sp, 16));
283 __ add(sp, sp, 32);
284 } else {
285 __ add(sp, sp, 16);
286 }
287
288 // Stub exit
289 __ b(*stub->continuation());
290 }
291
292 #undef __
293 #define __ sasm->
294
295 void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
296 DecoratorSet decorators) const {
297 __ prologue("zgc_load_barrier stub", false);
298
299 // We don't use push/pop_clobbered_registers() - we need to pull out the result from r0.
300 for (int i = 0; i < 32; i +=2) {
301 __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ pre(sp,-16)));
302 }
303
304 RegSet saveRegs = RegSet::range(r0,r28) - RegSet::of(r0);
305 __ push(saveRegs, sp);
306
307 // Setup arguments
308 __ load_parameter(0, c_rarg0);
309 __ load_parameter(1, c_rarg1);
310
311 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
312
313 __ pop(saveRegs, sp);
314
315 for (int i = 30; i >0; i -=2) {
316 __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ post(sp, 16)));
317 }
318
319 __ epilogue();
320 }
321 #endif // COMPILER1
322
323 #undef __
324 #define __ cgen->assembler()->
325
326 // Generates a register specific stub for calling
327 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
328 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
329 //
330 // The raddr register serves as both input and output for this stub. When the stub is
331 // called the raddr register contains the object field address (oop*) where the bad oop
332 // was loaded from, which caused the slow path to be taken. On return from the stub the
333 // raddr register contains the good/healed oop returned from
334 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
335 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
336 static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) {
337 // Don't generate stub for invalid registers
338 if (raddr == zr || raddr == r29 || raddr == r30) {
339 return NULL;
340 }
341
342 // Create stub name
343 char name[64];
344 const bool weak = (decorators & ON_WEAK_OOP_REF) != 0;
345 os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name());
346
347 __ align(CodeEntryAlignment);
348 StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode));
349 address start = __ pc();
350
351 // Save live registers
352 RegSet savedRegs = RegSet::range(r0,r18) - RegSet::of(raddr);
353
354 __ enter();
355 __ push(savedRegs, sp);
356
357 // Setup arguments
358 if (raddr != c_rarg1) {
359 __ mov(c_rarg1, raddr);
360 }
361
362 __ ldr(c_rarg0, Address(raddr));
363
364 // Call barrier function
365 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
366
367 // Move result returned in r0 to raddr, if needed
368 if (raddr != r0) {
369 __ mov(raddr, r0);
370 }
371
372 __ pop(savedRegs, sp);
373 __ leave();
374 __ ret(lr);
375
376 return start;
377 }
378
379 #undef __
380
381 static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) {
382 const int nregs = 28; // Exclude FP, XZR, SP from calculation.
383 const int code_size = nregs * 254; // Rough estimate of code size
384
385 ResourceMark rm;
386
387 CodeBuffer buf(BufferBlob::create(label, code_size));
388 StubCodeGenerator cgen(&buf);
389
390 for (int i = 0; i < nregs; i++) {
391 const Register reg = as_Register(i);
392 stub[i] = generate_load_barrier_stub(&cgen, reg, decorators);
393 }
394 }
395
396 void ZBarrierSetAssembler::barrier_stubs_init() {
397 barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub);
398 barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub);
399 }
400
401 address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) {
402 return _load_barrier_slow_stub[reg->encoding()];
403 }
404
405 address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) {
406 return _load_barrier_weak_slow_stub[reg->encoding()];
407 }