1 /*
2 * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "asm/macroAssembler.inline.hpp"
26 #include "code/codeBlob.hpp"
27 #include "gc/z/zBarrier.inline.hpp"
28 #include "gc/z/zBarrierSet.hpp"
29 #include "gc/z/zBarrierSetAssembler.hpp"
30 #include "gc/z/zBarrierSetRuntime.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "runtime/stubCodeGenerator.hpp"
33 #include "utilities/macros.hpp"
34 #ifdef COMPILER1
35 #include "c1/c1_LIRAssembler.hpp"
36 #include "c1/c1_MacroAssembler.hpp"
37 #include "gc/z/c1/zBarrierSetC1.hpp"
38 #endif // COMPILER1
39
40 ZBarrierSetAssembler::ZBarrierSetAssembler() :
41 _load_barrier_slow_stub(),
42 _load_barrier_weak_slow_stub() {}
43
44 #ifdef PRODUCT
45 #define BLOCK_COMMENT(str) /* nothing */
46 #else
47 #define BLOCK_COMMENT(str) __ block_comment(str)
48 #endif
49
50 #undef __
51 #define __ masm->
52
53 static void call_vm(MacroAssembler* masm,
54 address entry_point,
55 Register arg0,
56 Register arg1) {
57 // Setup arguments
58 if (arg1 == c_rarg0) {
59 if (arg0 == c_rarg1) {
60 __ xchgptr(c_rarg1, c_rarg0);
61 } else {
62 __ movptr(c_rarg1, arg1);
63 __ movptr(c_rarg0, arg0);
64 }
65 } else {
66 if (arg0 != c_rarg0) {
67 __ movptr(c_rarg0, arg0);
68 }
69 if (arg1 != c_rarg1) {
70 __ movptr(c_rarg1, arg1);
71 }
72 }
73
74 // Call VM
75 __ MacroAssembler::call_VM_leaf_base(entry_point, 2);
76 }
77
78 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
79 DecoratorSet decorators,
80 BasicType type,
81 Register dst,
82 Address src,
83 Register tmp1,
84 Register tmp_thread) {
85 if (!ZBarrierSet::barrier_needed(decorators, type)) {
86 // Barrier not needed
87 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
88 return;
89 }
90
91 BLOCK_COMMENT("ZBarrierSetAssembler::load_at {");
92
93 // Allocate scratch register
94 Register scratch = tmp1;
95 if (tmp1 == noreg) {
96 scratch = r12;
97 __ push(scratch);
98 }
99
100 assert_different_registers(dst, scratch);
101
102 Label done;
103
104 //
105 // Fast Path
106 //
107
108 // Load address
109 __ lea(scratch, src);
110
111 // Load oop at address
112 __ movptr(dst, Address(scratch, 0));
113
114 // Test address bad mask
115 __ testptr(dst, address_bad_mask_from_thread(r15_thread));
116 __ jcc(Assembler::zero, done);
117
118 //
119 // Slow path
120 //
121
122 // Save registers
123 __ push(rax);
124 __ push(rcx);
125 __ push(rdx);
126 __ push(rdi);
127 __ push(rsi);
128 __ push(r8);
129 __ push(r9);
130 __ push(r10);
131 __ push(r11);
132
133 // We may end up here from generate_native_wrapper, then the method may have
134 // floats as arguments, and we must spill them before calling the VM runtime
135 // leaf. From the interpreter all floats are passed on the stack.
136 assert(Argument::n_float_register_parameters_j == 8, "Assumption");
137 const int xmm_size = wordSize * 2;
138 const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j;
139 __ subptr(rsp, xmm_spill_size);
140 __ movdqu(Address(rsp, xmm_size * 7), xmm7);
141 __ movdqu(Address(rsp, xmm_size * 6), xmm6);
142 __ movdqu(Address(rsp, xmm_size * 5), xmm5);
143 __ movdqu(Address(rsp, xmm_size * 4), xmm4);
144 __ movdqu(Address(rsp, xmm_size * 3), xmm3);
145 __ movdqu(Address(rsp, xmm_size * 2), xmm2);
146 __ movdqu(Address(rsp, xmm_size * 1), xmm1);
147 __ movdqu(Address(rsp, xmm_size * 0), xmm0);
148
149 // Call VM
150 call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), dst, scratch);
151
152 // Restore registers
153 __ movdqu(xmm0, Address(rsp, xmm_size * 0));
154 __ movdqu(xmm1, Address(rsp, xmm_size * 1));
155 __ movdqu(xmm2, Address(rsp, xmm_size * 2));
156 __ movdqu(xmm3, Address(rsp, xmm_size * 3));
157 __ movdqu(xmm4, Address(rsp, xmm_size * 4));
158 __ movdqu(xmm5, Address(rsp, xmm_size * 5));
159 __ movdqu(xmm6, Address(rsp, xmm_size * 6));
160 __ movdqu(xmm7, Address(rsp, xmm_size * 7));
161 __ addptr(rsp, xmm_spill_size);
162
163 __ pop(r11);
164 __ pop(r10);
165 __ pop(r9);
166 __ pop(r8);
167 __ pop(rsi);
168 __ pop(rdi);
169 __ pop(rdx);
170 __ pop(rcx);
171
172 if (dst == rax) {
173 __ addptr(rsp, wordSize);
174 } else {
175 __ movptr(dst, rax);
176 __ pop(rax);
177 }
178
179 __ bind(done);
180
181 // Restore scratch register
182 if (tmp1 == noreg) {
183 __ pop(scratch);
184 }
185
186 BLOCK_COMMENT("} ZBarrierSetAssembler::load_at");
187 }
188
189 #ifdef ASSERT
190
191 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
192 DecoratorSet decorators,
193 BasicType type,
194 Address dst,
195 Register src,
196 Register tmp1,
197 Register tmp2) {
198 BLOCK_COMMENT("ZBarrierSetAssembler::store_at {");
199
200 // Verify oop store
201 if (type == T_OBJECT || type == T_ARRAY) {
202 // Note that src could be noreg, which means we
203 // are storing null and can skip verification.
204 if (src != noreg) {
205 Label done;
206 __ testptr(src, address_bad_mask_from_thread(r15_thread));
207 __ jcc(Assembler::zero, done);
208 __ stop("Verify oop store failed");
209 __ should_not_reach_here();
210 __ bind(done);
211 }
212 }
213
214 // Store value
215 BarrierSetAssembler::store_at(masm, decorators, type, dst, src, tmp1, tmp2);
216
217 BLOCK_COMMENT("} ZBarrierSetAssembler::store_at");
218 }
219
220 #endif // ASSERT
221
222 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
223 DecoratorSet decorators,
224 BasicType type,
225 Register src,
226 Register dst,
227 Register count) {
228 if (!ZBarrierSet::barrier_needed(decorators, type)) {
229 // Barrier not needed
230 return;
231 }
232
233 BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
234
235 // Save registers
236 __ pusha();
237
238 // Call VM
239 call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), src, count);
240
241 // Restore registers
242 __ popa();
243
244 BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
245 }
246
247 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
248 Register jni_env,
249 Register obj,
250 Register tmp,
251 Label& slowpath) {
252 BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
253
254 // Resolve jobject
255 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
256
257 // Test address bad mask
258 __ testptr(obj, address_bad_mask_from_jni_env(jni_env));
259 __ jcc(Assembler::notZero, slowpath);
260
261 BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
262 }
263
264 #ifdef COMPILER1
265
266 #undef __
267 #define __ ce->masm()->
268
269 void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
270 LIR_Opr ref) const {
271 __ testptr(ref->as_register(), address_bad_mask_from_thread(r15_thread));
272 }
273
274 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
275 ZLoadBarrierStubC1* stub) const {
276 // Stub entry
277 __ bind(*stub->entry());
278
279 Register ref = stub->ref()->as_register();
280 Register ref_addr = noreg;
281
282 if (stub->tmp()->is_valid()) {
283 // Load address into tmp register
284 ce->leal(stub->ref_addr(), stub->tmp());
285 ref_addr = stub->tmp()->as_pointer_register();
286 } else {
287 // Address already in register
288 ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
289 }
290
291 assert_different_registers(ref, ref_addr, noreg);
292
293 // Save rax unless it is the result register
294 if (ref != rax) {
295 __ push(rax);
296 }
297
298 // Setup arguments and call runtime stub
299 __ subptr(rsp, 2 * BytesPerWord);
300 ce->store_parameter(ref_addr, 1);
301 ce->store_parameter(ref, 0);
302 __ call(RuntimeAddress(stub->runtime_stub()));
303 __ addptr(rsp, 2 * BytesPerWord);
304
305 // Verify result
306 __ verify_oop(rax, "Bad oop");
307
308 // Restore rax unless it is the result register
309 if (ref != rax) {
310 __ movptr(ref, rax);
311 __ pop(rax);
312 }
313
314 // Stub exit
315 __ jmp(*stub->continuation());
316 }
317
318 #undef __
319 #define __ sasm->
320
321 void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
322 DecoratorSet decorators) const {
323 // Enter and save registers
324 __ enter();
325 __ save_live_registers_no_oop_map(true /* save_fpu_registers */);
326
327 // Setup arguments
328 __ load_parameter(1, c_rarg1);
329 __ load_parameter(0, c_rarg0);
330
331 // Call VM
332 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
333
334 // Restore registers and return
335 __ restore_live_registers_except_rax(true /* restore_fpu_registers */);
336 __ leave();
337 __ ret(0);
338 }
339
340 #endif // COMPILER1
341
342 #undef __
343 #define __ cgen->assembler()->
344
345 // Generates a register specific stub for calling
346 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
347 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
348 //
349 // The raddr register serves as both input and output for this stub. When the stub is
350 // called the raddr register contains the object field address (oop*) where the bad oop
351 // was loaded from, which caused the slow path to be taken. On return from the stub the
352 // raddr register contains the good/healed oop returned from
353 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
354 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
355 static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) {
356 // Don't generate stub for invalid registers
357 if (raddr == rsp || raddr == r12 || raddr == r15) {
358 return NULL;
359 }
360
361 // Create stub name
362 char name[64];
363 const bool weak = (decorators & ON_WEAK_OOP_REF) != 0;
364 os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name());
365
366 __ align(CodeEntryAlignment);
367 StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode));
368 address start = __ pc();
369
370 // Save live registers
371 if (raddr != rax) {
372 __ push(rax);
373 }
374 if (raddr != rcx) {
375 __ push(rcx);
376 }
377 if (raddr != rdx) {
378 __ push(rdx);
379 }
380 if (raddr != rsi) {
381 __ push(rsi);
382 }
383 if (raddr != rdi) {
384 __ push(rdi);
385 }
386 if (raddr != r8) {
387 __ push(r8);
388 }
389 if (raddr != r9) {
390 __ push(r9);
391 }
392 if (raddr != r10) {
393 __ push(r10);
394 }
395 if (raddr != r11) {
396 __ push(r11);
397 }
398
399 // Setup arguments
400 if (raddr != c_rarg1) {
401 __ movq(c_rarg1, raddr);
402 }
403 __ movq(c_rarg0, Address(raddr, 0));
404
405 // Call barrier function
406 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
407
408 // Move result returned in rax to raddr, if needed
409 if (raddr != rax) {
410 __ movq(raddr, rax);
411 }
412
413 // Restore saved registers
414 if (raddr != r11) {
415 __ pop(r11);
416 }
417 if (raddr != r10) {
418 __ pop(r10);
419 }
420 if (raddr != r9) {
421 __ pop(r9);
422 }
423 if (raddr != r8) {
424 __ pop(r8);
425 }
426 if (raddr != rdi) {
427 __ pop(rdi);
428 }
429 if (raddr != rsi) {
430 __ pop(rsi);
431 }
432 if (raddr != rdx) {
433 __ pop(rdx);
434 }
435 if (raddr != rcx) {
436 __ pop(rcx);
437 }
438 if (raddr != rax) {
439 __ pop(rax);
440 }
441
442 __ ret(0);
443
444 return start;
445 }
446
447 #undef __
448
449 static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) {
450 const int nregs = RegisterImpl::number_of_registers;
451 const int code_size = nregs * 128; // Rough estimate of code size
452
453 ResourceMark rm;
454
455 CodeBuffer buf(BufferBlob::create(label, code_size));
456 StubCodeGenerator cgen(&buf);
457
458 for (int i = 0; i < nregs; i++) {
459 const Register reg = as_Register(i);
460 stub[i] = generate_load_barrier_stub(&cgen, reg, decorators);
461 }
462 }
463
464 void ZBarrierSetAssembler::barrier_stubs_init() {
465 barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub);
466 barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub);
467 }
468
469 address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) {
470 return _load_barrier_slow_stub[reg->encoding()];
471 }
472
473 address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) {
474 return _load_barrier_weak_slow_stub[reg->encoding()];
475 }