1 /* 2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "asm/macroAssembler.inline.hpp" 26 #include "code/codeBlob.hpp" 27 #include "gc/z/zBarrier.inline.hpp" 28 #include "gc/z/zBarrierSet.hpp" 29 #include "gc/z/zBarrierSetAssembler.hpp" 30 #include "gc/z/zBarrierSetRuntime.hpp" 31 #include "memory/resourceArea.hpp" 32 #ifdef COMPILER1 33 #include "c1/c1_LIRAssembler.hpp" 34 #include "c1/c1_MacroAssembler.hpp" 35 #include "gc/z/c1/zBarrierSetC1.hpp" 36 #endif // COMPILER1 37 38 #include "gc/z/zThreadLocalData.hpp" 39 40 ZBarrierSetAssembler::ZBarrierSetAssembler() : 41 _load_barrier_slow_stub(), 42 _load_barrier_weak_slow_stub() {} 43 44 #ifdef PRODUCT 45 #define BLOCK_COMMENT(str) /* nothing */ 46 #else 47 #define BLOCK_COMMENT(str) __ block_comment(str) 48 #endif 49 50 #undef __ 51 #define __ masm-> 52 53 void ZBarrierSetAssembler::load_at(MacroAssembler* masm, 54 DecoratorSet decorators, 55 BasicType type, 56 Register dst, 57 Address src, 58 Register tmp1, 59 Register tmp_thread) { 60 if (!ZBarrierSet::barrier_needed(decorators, type)) { 61 // Barrier not needed 62 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 63 return; 64 } 65 66 // rscratch1 can be passed as src or dst, so don't use it. 67 RegSet savedRegs = RegSet::of(rscratch2, rheapbase); 68 69 Label done; 70 assert_different_registers(rheapbase, rscratch2, dst); 71 assert_different_registers(rheapbase, rscratch2, src.base()); 72 73 __ push(savedRegs, sp); 74 75 // Load bad mask into scratch register. 76 __ ldr(rheapbase, address_bad_mask_from_thread(rthread)); 77 __ lea(rscratch2, src); 78 __ ldr(dst, src); 79 80 // Test reference against bad mask. If mask bad, then we need to fix it up. 81 __ tst(dst, rheapbase); 82 __ br(Assembler::EQ, done); 83 84 __ enter(); 85 86 __ push(RegSet::range(r0,r28) - RegSet::of(dst), sp); 87 88 if (c_rarg0 != dst) { 89 __ mov(c_rarg0, dst); 90 } 91 __ mov(c_rarg1, rscratch2); 92 93 int step = 4 * wordSize; 94 __ mov(rscratch1, -step); 95 __ sub(sp, sp, step); 96 97 for (int i = 28; i >= 4; i -= 4) { 98 __ st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), 99 as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch1))); 100 } 101 102 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); 103 104 for (int i = 0; i <= 28; i += 4) { 105 __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), 106 as_FloatRegister(i+3), __ T1D, Address(__ post(sp, step))); 107 } 108 109 // Make sure dst has the return value. 110 if (dst != r0) { 111 __ mov(dst, r0); 112 } 113 114 __ pop(RegSet::range(r0,r28) - RegSet::of(dst), sp); 115 __ leave(); 116 117 __ bind(done); 118 119 // Restore tmps 120 __ pop(savedRegs, sp); 121 } 122 123 #ifdef ASSERT 124 125 void ZBarrierSetAssembler::store_at(MacroAssembler* masm, 126 DecoratorSet decorators, 127 BasicType type, 128 Address dst, 129 Register val, 130 Register tmp1, 131 Register tmp2) { 132 // Verify value 133 if (type == T_OBJECT || type == T_ARRAY) { 134 // Note that src could be noreg, which means we 135 // are storing null and can skip verification. 136 if (val != noreg) { 137 Label done; 138 139 // tmp1 and tmp2 are often set to noreg. 140 RegSet savedRegs = RegSet::of(rscratch1); 141 __ push(savedRegs, sp); 142 143 __ ldr(rscratch1, address_bad_mask_from_thread(rthread)); 144 __ tst(val, rscratch1); 145 __ br(Assembler::EQ, done); 146 __ stop("Verify oop store failed"); 147 __ should_not_reach_here(); 148 __ bind(done); 149 __ pop(savedRegs, sp); 150 } 151 } 152 153 // Store value 154 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); 155 } 156 157 #endif // ASSERT 158 159 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, 160 DecoratorSet decorators, 161 bool is_oop, 162 Register src, 163 Register dst, 164 Register count, 165 RegSet saved_regs) { 166 if (!is_oop) { 167 // Barrier not needed 168 return; 169 } 170 171 BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {"); 172 173 assert_different_registers(src, count, rscratch1); 174 175 __ pusha(); 176 177 if (count == c_rarg0) { 178 if (src == c_rarg1) { 179 // exactly backwards!! 180 __ mov(rscratch1, c_rarg0); 181 __ mov(c_rarg0, c_rarg1); 182 __ mov(c_rarg1, rscratch1); 183 } else { 184 __ mov(c_rarg1, count); 185 __ mov(c_rarg0, src); 186 } 187 } else { 188 __ mov(c_rarg0, src); 189 __ mov(c_rarg1, count); 190 } 191 192 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2); 193 194 __ popa(); 195 BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue"); 196 } 197 198 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, 199 Register jni_env, 200 Register robj, 201 Register tmp, 202 Label& slowpath) { 203 BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {"); 204 205 assert_different_registers(jni_env, robj, tmp); 206 207 // Resolve jobject 208 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath); 209 210 // The Address offset is too large to direct load - -784. Our range is +127, -128. 211 __ mov(tmp, (long int)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) - 212 in_bytes(JavaThread::jni_environment_offset()))); 213 // Load address bad mask 214 __ add(tmp, jni_env, tmp); 215 __ ldr(tmp, Address(tmp)); 216 217 // Check address bad mask 218 __ tst(robj, tmp); 219 __ br(Assembler::NE, slowpath); 220 221 BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native"); 222 } 223 224 #ifdef COMPILER1 225 226 #undef __ 227 #define __ ce->masm()-> 228 229 void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce, 230 LIR_Opr ref) const { 231 assert_different_registers(rheapbase, rthread, ref->as_register()); 232 233 __ ldr(rheapbase, address_bad_mask_from_thread(rthread)); 234 __ tst(ref->as_register(), rheapbase); 235 } 236 237 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, 238 ZLoadBarrierStubC1* stub) const { 239 // Stub entry 240 __ bind(*stub->entry()); 241 242 Register ref = stub->ref()->as_register(); 243 Register ref_addr = noreg; 244 Register tmp = noreg; 245 246 if (stub->tmp()->is_valid()) { 247 // Load address into tmp register 248 ce->leal(stub->ref_addr(), stub->tmp()); 249 ref_addr = tmp = stub->tmp()->as_pointer_register(); 250 } else { 251 // Address already in register 252 ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register(); 253 } 254 255 assert_different_registers(ref, ref_addr, noreg); 256 257 // Save r0 unless it is the result or tmp register 258 // Set up SP to accomodate parameters and maybe r0.. 259 if (ref != r0 && tmp != r0) { 260 __ sub(sp, sp, 32); 261 __ str(r0, Address(sp, 16)); 262 } else { 263 __ sub(sp, sp, 16); 264 } 265 266 // Setup arguments and call runtime stub 267 ce->store_parameter(ref_addr, 1); 268 ce->store_parameter(ref, 0); 269 270 __ far_call(stub->runtime_stub()); 271 272 // Verify result 273 __ verify_oop(r0, "Bad oop"); 274 275 // Move result into place 276 if (ref != r0) { 277 __ mov(ref, r0); 278 } 279 280 // Restore r0 unless it is the result or tmp register 281 if (ref != r0 && tmp != r0) { 282 __ ldr(r0, Address(sp, 16)); 283 __ add(sp, sp, 32); 284 } else { 285 __ add(sp, sp, 16); 286 } 287 288 // Stub exit 289 __ b(*stub->continuation()); 290 } 291 292 #undef __ 293 #define __ sasm-> 294 295 void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, 296 DecoratorSet decorators) const { 297 __ prologue("zgc_load_barrier stub", false); 298 299 // We don't use push/pop_clobbered_registers() - we need to pull out the result from r0. 300 for (int i = 0; i < 32; i +=2) { 301 __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ pre(sp,-16))); 302 } 303 304 RegSet saveRegs = RegSet::range(r0,r28) - RegSet::of(r0); 305 __ push(saveRegs, sp); 306 307 // Setup arguments 308 __ load_parameter(0, c_rarg0); 309 __ load_parameter(1, c_rarg1); 310 311 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); 312 313 __ pop(saveRegs, sp); 314 315 for (int i = 30; i >0; i -=2) { 316 __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ post(sp, 16))); 317 } 318 319 __ epilogue(); 320 } 321 #endif // COMPILER1 322 323 #undef __ 324 #define __ cgen->assembler()-> 325 326 // Generates a register specific stub for calling 327 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or 328 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). 329 // 330 // The raddr register serves as both input and output for this stub. When the stub is 331 // called the raddr register contains the object field address (oop*) where the bad oop 332 // was loaded from, which caused the slow path to be taken. On return from the stub the 333 // raddr register contains the good/healed oop returned from 334 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or 335 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). 336 static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { 337 // Don't generate stub for invalid registers 338 if (raddr == zr || raddr == r29 || raddr == r30) { 339 return NULL; 340 } 341 342 // Create stub name 343 char name[64]; 344 const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; 345 os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); 346 347 __ align(CodeEntryAlignment); 348 StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); 349 address start = __ pc(); 350 351 // Save live registers 352 RegSet savedRegs = RegSet::range(r0,r18) - RegSet::of(raddr); 353 354 __ enter(); 355 __ push(savedRegs, sp); 356 357 // Setup arguments 358 if (raddr != c_rarg1) { 359 __ mov(c_rarg1, raddr); 360 } 361 362 __ ldr(c_rarg0, Address(raddr)); 363 364 // Call barrier function 365 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); 366 367 // Move result returned in r0 to raddr, if needed 368 if (raddr != r0) { 369 __ mov(raddr, r0); 370 } 371 372 __ pop(savedRegs, sp); 373 __ leave(); 374 __ ret(lr); 375 376 return start; 377 } 378 379 #undef __ 380 381 static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) { 382 const int nregs = 28; // Exclude FP, XZR, SP from calculation. 383 const int code_size = nregs * 254; // Rough estimate of code size 384 385 ResourceMark rm; 386 387 CodeBuffer buf(BufferBlob::create(label, code_size)); 388 StubCodeGenerator cgen(&buf); 389 390 for (int i = 0; i < nregs; i++) { 391 const Register reg = as_Register(i); 392 stub[i] = generate_load_barrier_stub(&cgen, reg, decorators); 393 } 394 } 395 396 void ZBarrierSetAssembler::barrier_stubs_init() { 397 barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub); 398 barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub); 399 } 400 401 address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) { 402 return _load_barrier_slow_stub[reg->encoding()]; 403 } 404 405 address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) { 406 return _load_barrier_weak_slow_stub[reg->encoding()]; 407 }