1 /*
2 * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "asm/macroAssembler.inline.hpp"
26 #include "code/codeBlob.hpp"
27 #include "code/vmreg.inline.hpp"
28 #include "gc/z/zBarrier.inline.hpp"
29 #include "gc/z/zBarrierSet.hpp"
30 #include "gc/z/zBarrierSetAssembler.hpp"
31 #include "gc/z/zBarrierSetRuntime.hpp"
32 #include "gc/z/zThreadLocalData.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/z/c1/zBarrierSetC1.hpp"
40 #endif // COMPILER1
41 #ifdef COMPILER2
42 #include "gc/z/c2/zBarrierSetC2.hpp"
43 #endif // COMPILER2
44
45 #ifdef PRODUCT
46 #define BLOCK_COMMENT(str) /* nothing */
47 #else
48 #define BLOCK_COMMENT(str) __ block_comment(str)
49 #endif
50
51 #undef __
52 #define __ masm->
53
54 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
55 DecoratorSet decorators,
56 BasicType type,
57 Register dst,
58 Address src,
59 Register tmp1,
60 Register tmp_thread) {
61 if (!ZBarrierSet::barrier_needed(decorators, type)) {
62 // Barrier not needed
63 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
64 return;
65 }
66
67 assert_different_registers(rscratch1, rscratch2, src.base());
68 assert_different_registers(rscratch1, rscratch2, dst);
69
70 Label done;
71
72 // Load bad mask into scratch register.
73 __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
74 __ lea(rscratch2, src);
75 __ ldr(dst, src);
76
77 // Test reference against bad mask. If mask bad, then we need to fix it up.
78 __ tst(dst, rscratch1);
79 __ br(Assembler::EQ, done);
80
81 __ enter();
82
83 __ push_call_clobbered_registers_except(RegSet::of(dst));
84
85 if (c_rarg0 != dst) {
86 __ mov(c_rarg0, dst);
87 }
88 __ mov(c_rarg1, rscratch2);
89
90 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
91
92 // Make sure dst has the return value.
93 if (dst != r0) {
94 __ mov(dst, r0);
95 }
96
97 __ pop_call_clobbered_registers_except(RegSet::of(dst));
98 __ leave();
99
100 __ bind(done);
101 }
102
103 #ifdef ASSERT
104
105 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
106 DecoratorSet decorators,
107 BasicType type,
108 Address dst,
109 Register val,
110 Register tmp1,
111 Register tmp2) {
112 // Verify value
113 if (is_reference_type(type)) {
114 // Note that src could be noreg, which means we
115 // are storing null and can skip verification.
116 if (val != noreg) {
117 Label done;
118
119 // tmp1 and tmp2 are often set to noreg.
120 RegSet savedRegs = RegSet::of(rscratch1);
121 __ push(savedRegs, sp);
122
123 __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
124 __ tst(val, rscratch1);
125 __ br(Assembler::EQ, done);
126 __ stop("Verify oop store failed");
127 __ should_not_reach_here();
128 __ bind(done);
129 __ pop(savedRegs, sp);
130 }
131 }
132
133 // Store value
134 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
135 }
136
137 #endif // ASSERT
138
139 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
140 DecoratorSet decorators,
141 bool is_oop,
142 Register src,
143 Register dst,
144 Register count,
145 RegSet saved_regs) {
146 if (!is_oop) {
147 // Barrier not needed
148 return;
149 }
150
151 BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
152
153 assert_different_registers(src, count, rscratch1);
154
155 __ push(saved_regs, sp);
156
157 if (count == c_rarg0) {
158 if (src == c_rarg1) {
159 // exactly backwards!!
160 __ mov(rscratch1, c_rarg0);
161 __ mov(c_rarg0, c_rarg1);
162 __ mov(c_rarg1, rscratch1);
163 } else {
164 __ mov(c_rarg1, count);
165 __ mov(c_rarg0, src);
166 }
167 } else {
168 __ mov(c_rarg0, src);
169 __ mov(c_rarg1, count);
170 }
171
172 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
173
174 __ pop(saved_regs, sp);
175
176 BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
177 }
178
179 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
180 Register jni_env,
181 Register robj,
182 Register tmp,
183 Label& slowpath) {
184 BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
185
186 assert_different_registers(jni_env, robj, tmp);
187
188 // Resolve jobject
189 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
190
191 // The Address offset is too large to direct load - -784. Our range is +127, -128.
192 __ mov(tmp, (int64_t)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) -
193 in_bytes(JavaThread::jni_environment_offset())));
194
195 // Load address bad mask
196 __ add(tmp, jni_env, tmp);
197 __ ldr(tmp, Address(tmp));
198
199 // Check address bad mask
200 __ tst(robj, tmp);
201 __ br(Assembler::NE, slowpath);
202
203 BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
204 }
205
206 #ifdef COMPILER1
207
208 #undef __
209 #define __ ce->masm()->
210
211 void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
212 LIR_Opr ref) const {
213 assert_different_registers(rscratch1, rthread, ref->as_register());
214
215 __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
216 __ tst(ref->as_register(), rscratch1);
217 }
218
219 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
220 ZLoadBarrierStubC1* stub) const {
221 // Stub entry
222 __ bind(*stub->entry());
223
224 Register ref = stub->ref()->as_register();
225 Register ref_addr = noreg;
226 Register tmp = noreg;
227
228 if (stub->tmp()->is_valid()) {
229 // Load address into tmp register
230 ce->leal(stub->ref_addr(), stub->tmp());
231 ref_addr = tmp = stub->tmp()->as_pointer_register();
232 } else {
233 // Address already in register
234 ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
235 }
236
237 assert_different_registers(ref, ref_addr, noreg);
238
239 // Save r0 unless it is the result or tmp register
240 // Set up SP to accomodate parameters and maybe r0..
241 if (ref != r0 && tmp != r0) {
242 __ sub(sp, sp, 32);
243 __ str(r0, Address(sp, 16));
244 } else {
245 __ sub(sp, sp, 16);
246 }
247
248 // Setup arguments and call runtime stub
249 ce->store_parameter(ref_addr, 1);
250 ce->store_parameter(ref, 0);
251
252 __ far_call(stub->runtime_stub());
253
254 // Verify result
255 __ verify_oop(r0, "Bad oop");
256
257 // Move result into place
258 if (ref != r0) {
259 __ mov(ref, r0);
260 }
261
262 // Restore r0 unless it is the result or tmp register
263 if (ref != r0 && tmp != r0) {
264 __ ldr(r0, Address(sp, 16));
265 __ add(sp, sp, 32);
266 } else {
267 __ add(sp, sp, 16);
268 }
269
270 // Stub exit
271 __ b(*stub->continuation());
272 }
273
274 #undef __
275 #define __ sasm->
276
277 void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
278 DecoratorSet decorators) const {
279 __ prologue("zgc_load_barrier stub", false);
280
281 __ push_call_clobbered_registers_except(RegSet::of(r0));
282
283 // Setup arguments
284 __ load_parameter(0, c_rarg0);
285 __ load_parameter(1, c_rarg1);
286
287 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
288
289 __ pop_call_clobbered_registers_except(RegSet::of(r0));
290
291 __ epilogue();
292 }
293 #endif // COMPILER1
294
295 #ifdef COMPILER2
296
297 OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
298 if (!OptoReg::is_reg(opto_reg)) {
299 return OptoReg::Bad;
300 }
301
302 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
303 if (vm_reg->is_FloatRegister()) {
304 return opto_reg & ~1;
305 }
306
307 return opto_reg;
308 }
309
310 #undef __
311 #define __ _masm->
312
313 class ZSaveLiveRegisters {
314 private:
315 MacroAssembler* const _masm;
316 RegSet _gp_regs;
317 RegSet _fp_regs;
318
319 public:
320 void initialize(ZLoadBarrierStubC2* stub) {
321 // Create mask of live registers
322 RegMask live = stub->live();
323
324 // Record registers that needs to be saved/restored
325 while (live.is_NotEmpty()) {
326 const OptoReg::Name opto_reg = live.find_first_elem();
327 live.Remove(opto_reg);
328 if (OptoReg::is_reg(opto_reg)) {
329 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
330 if (vm_reg->is_Register()) {
331 _gp_regs += RegSet::of(vm_reg->as_Register());
332 } else if (vm_reg->is_FloatRegister()) {
333 _fp_regs += RegSet::of((Register)vm_reg->as_FloatRegister());
334 } else {
335 fatal("Unknown register type");
336 }
337 }
338 }
339
340 // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
341 _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref());
342 }
343
344 ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
345 _masm(masm),
346 _gp_regs(),
347 _fp_regs() {
348
349 // Figure out what registers to save/restore
350 initialize(stub);
351
352 // Save registers
353 __ push(_gp_regs, sp);
354 __ push_fp(_fp_regs, sp);
355 }
356
357 ~ZSaveLiveRegisters() {
358 // Restore registers
359 __ pop_fp(_fp_regs, sp);
360 __ pop(_gp_regs, sp);
361 }
362 };
363
364 #undef __
365 #define __ _masm->
366
367 class ZSetupArguments {
368 private:
369 MacroAssembler* const _masm;
370 const Register _ref;
371 const Address _ref_addr;
372
373 public:
374 ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
375 _masm(masm),
376 _ref(stub->ref()),
377 _ref_addr(stub->ref_addr()) {
378
379 // Setup arguments
380 if (_ref_addr.base() == noreg) {
381 // No self healing
382 if (_ref != c_rarg0) {
383 __ mov(c_rarg0, _ref);
384 }
385 __ mov(c_rarg1, 0);
386 } else {
387 // Self healing
388 if (_ref == c_rarg0) {
389 // _ref is already at correct place
390 __ lea(c_rarg1, _ref_addr);
391 } else if (_ref != c_rarg1) {
392 // _ref is in wrong place, but not in c_rarg1, so fix it first
393 __ lea(c_rarg1, _ref_addr);
394 __ mov(c_rarg0, _ref);
395 } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
396 assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
397 __ mov(c_rarg0, _ref);
398 __ lea(c_rarg1, _ref_addr);
399 } else {
400 assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
401 if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) {
402 __ mov(rscratch2, c_rarg1);
403 __ lea(c_rarg1, _ref_addr);
404 __ mov(c_rarg0, rscratch2);
405 } else {
406 ShouldNotReachHere();
407 }
408 }
409 }
410 }
411
412 ~ZSetupArguments() {
413 // Transfer result
414 if (_ref != r0) {
415 __ mov(_ref, r0);
416 }
417 }
418 };
419
420 #undef __
421 #define __ masm->
422
423 void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
424 BLOCK_COMMENT("ZLoadBarrierStubC2");
425
426 // Stub entry
427 __ bind(*stub->entry());
428
429 {
430 ZSaveLiveRegisters save_live_registers(masm, stub);
431 ZSetupArguments setup_arguments(masm, stub);
432 __ mov(rscratch1, stub->slow_path());
433 __ blr(rscratch1);
434 if (UseSVE > 0) {
435 // Reinitialize the ptrue predicate register, in case the external runtime
436 // call clobbers ptrue reg, as we may return to SVE compiled code.
437 __ reinitialize_ptrue();
438 }
439 }
440 // Stub exit
441 __ b(*stub->continuation());
442 }
443
444 #undef __
445
446 #endif // COMPILER2