1 /*
   2  * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "asm/macroAssembler.inline.hpp"
  26 #include "code/codeBlob.hpp"
  27 #include "gc/z/zBarrier.inline.hpp"
  28 #include "gc/z/zBarrierSet.hpp"
  29 #include "gc/z/zBarrierSetAssembler.hpp"
  30 #include "gc/z/zBarrierSetRuntime.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "runtime/stubCodeGenerator.hpp"
  33 #include "utilities/macros.hpp"
  34 #ifdef COMPILER1
  35 #include "c1/c1_LIRAssembler.hpp"
  36 #include "c1/c1_MacroAssembler.hpp"
  37 #include "gc/z/c1/zBarrierSetC1.hpp"
  38 #endif // COMPILER1
  39 
  40 ZBarrierSetAssembler::ZBarrierSetAssembler() :
  41     _load_barrier_slow_stub(),
  42     _load_barrier_weak_slow_stub() {}
  43 
  44 #ifdef PRODUCT
  45 #define BLOCK_COMMENT(str) /* nothing */
  46 #else
  47 #define BLOCK_COMMENT(str) __ block_comment(str)
  48 #endif
  49 
  50 #undef __
  51 #define __ masm->
  52 
  53 static void call_vm(MacroAssembler* masm,
  54                     address entry_point,
  55                     Register arg0,
  56                     Register arg1) {
  57   // Setup arguments
  58   if (arg1 == c_rarg0) {
  59     if (arg0 == c_rarg1) {
  60       __ xchgptr(c_rarg1, c_rarg0);
  61     } else {
  62       __ movptr(c_rarg1, arg1);
  63       __ movptr(c_rarg0, arg0);
  64     }
  65   } else {
  66     if (arg0 != c_rarg0) {
  67       __ movptr(c_rarg0, arg0);
  68     }
  69     if (arg1 != c_rarg1) {
  70       __ movptr(c_rarg1, arg1);
  71     }
  72   }
  73 
  74   // Call VM
  75   __ MacroAssembler::call_VM_leaf_base(entry_point, 2);
  76 }
  77 
  78 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
  79                                    DecoratorSet decorators,
  80                                    BasicType type,
  81                                    Register dst,
  82                                    Address src,
  83                                    Register tmp1,
  84                                    Register tmp_thread) {
  85   if (!ZBarrierSet::barrier_needed(decorators, type)) {
  86     // Barrier not needed
  87     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
  88     return;
  89   }
  90 
  91   BLOCK_COMMENT("ZBarrierSetAssembler::load_at {");
  92 
  93   // Allocate scratch register
  94   Register scratch = tmp1;
  95   if (tmp1 == noreg) {
  96     scratch = r12;
  97     __ push(scratch);
  98   }
  99 
 100   assert_different_registers(dst, scratch);
 101 
 102   Label done;
 103 
 104   //
 105   // Fast Path
 106   //
 107 
 108   // Load address
 109   __ lea(scratch, src);
 110 
 111   // Load oop at address
 112   __ movptr(dst, Address(scratch, 0));
 113 
 114   // Test address bad mask
 115   __ testptr(dst, address_bad_mask_from_thread(r15_thread));
 116   __ jcc(Assembler::zero, done);
 117 
 118   //
 119   // Slow path
 120   //
 121 
 122   // Save registers
 123   __ push(rax);
 124   __ push(rcx);
 125   __ push(rdx);
 126   __ push(rdi);
 127   __ push(rsi);
 128   __ push(r8);
 129   __ push(r9);
 130   __ push(r10);
 131   __ push(r11);
 132 
 133   // We may end up here from generate_native_wrapper, then the method may have
 134   // floats as arguments, and we must spill them before calling the VM runtime
 135   // leaf. From the interpreter all floats are passed on the stack.
 136   assert(Argument::n_float_register_parameters_j == 8, "Assumption");
 137   const int xmm_size = wordSize * 2;
 138   const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j;
 139   __ subptr(rsp, xmm_spill_size);
 140   __ movdqu(Address(rsp, xmm_size * 7), xmm7);
 141   __ movdqu(Address(rsp, xmm_size * 6), xmm6);
 142   __ movdqu(Address(rsp, xmm_size * 5), xmm5);
 143   __ movdqu(Address(rsp, xmm_size * 4), xmm4);
 144   __ movdqu(Address(rsp, xmm_size * 3), xmm3);
 145   __ movdqu(Address(rsp, xmm_size * 2), xmm2);
 146   __ movdqu(Address(rsp, xmm_size * 1), xmm1);
 147   __ movdqu(Address(rsp, xmm_size * 0), xmm0);
 148 
 149   // Call VM
 150   call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), dst, scratch);
 151 
 152   // Restore registers
 153   __ movdqu(xmm0, Address(rsp, xmm_size * 0));
 154   __ movdqu(xmm1, Address(rsp, xmm_size * 1));
 155   __ movdqu(xmm2, Address(rsp, xmm_size * 2));
 156   __ movdqu(xmm3, Address(rsp, xmm_size * 3));
 157   __ movdqu(xmm4, Address(rsp, xmm_size * 4));
 158   __ movdqu(xmm5, Address(rsp, xmm_size * 5));
 159   __ movdqu(xmm6, Address(rsp, xmm_size * 6));
 160   __ movdqu(xmm7, Address(rsp, xmm_size * 7));
 161   __ addptr(rsp, xmm_spill_size);
 162 
 163   __ pop(r11);
 164   __ pop(r10);
 165   __ pop(r9);
 166   __ pop(r8);
 167   __ pop(rsi);
 168   __ pop(rdi);
 169   __ pop(rdx);
 170   __ pop(rcx);
 171 
 172   if (dst == rax) {
 173     __ addptr(rsp, wordSize);
 174   } else {
 175     __ movptr(dst, rax);
 176     __ pop(rax);
 177   }
 178 
 179   __ bind(done);
 180 
 181   // Restore scratch register
 182   if (tmp1 == noreg) {
 183     __ pop(scratch);
 184   }
 185 
 186   BLOCK_COMMENT("} ZBarrierSetAssembler::load_at");
 187 }
 188 
 189 #ifdef ASSERT
 190 
 191 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
 192                                     DecoratorSet decorators,
 193                                     BasicType type,
 194                                     Address dst,
 195                                     Register src,
 196                                     Register tmp1,
 197                                     Register tmp2) {
 198   BLOCK_COMMENT("ZBarrierSetAssembler::store_at {");
 199 
 200   // Verify oop store
 201   if (type == T_OBJECT || type == T_ARRAY) {
 202     // Note that src could be noreg, which means we
 203     // are storing null and can skip verification.
 204     if (src != noreg) {
 205       Label done;
 206       __ testptr(src, address_bad_mask_from_thread(r15_thread));
 207       __ jcc(Assembler::zero, done);
 208       __ stop("Verify oop store failed");
 209       __ should_not_reach_here();
 210       __ bind(done);
 211     }
 212   }
 213 
 214   // Store value
 215   BarrierSetAssembler::store_at(masm, decorators, type, dst, src, tmp1, tmp2);
 216 
 217   BLOCK_COMMENT("} ZBarrierSetAssembler::store_at");
 218 }
 219 
 220 #endif // ASSERT
 221 
 222 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
 223                                               DecoratorSet decorators,
 224                                               BasicType type,
 225                                               Register src,
 226                                               Register dst,
 227                                               Register count) {
 228   if (!ZBarrierSet::barrier_needed(decorators, type)) {
 229     // Barrier not needed
 230     return;
 231   }
 232 
 233   BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
 234 
 235   // Save registers
 236   __ pusha();
 237 
 238   // Call VM
 239   call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), src, count);
 240 
 241   // Restore registers
 242   __ popa();
 243 
 244   BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
 245 }
 246 
 247 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
 248                                                          Register jni_env,
 249                                                          Register obj,
 250                                                          Register tmp,
 251                                                          Label& slowpath) {
 252   BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
 253 
 254   // Resolve jobject
 255   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
 256 
 257   // Test address bad mask
 258   __ testptr(obj, address_bad_mask_from_jni_env(jni_env));
 259   __ jcc(Assembler::notZero, slowpath);
 260 
 261   BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
 262 }
 263 
 264 #ifdef COMPILER1
 265 
 266 #undef __
 267 #define __ ce->masm()->
 268 
 269 void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
 270                                                          LIR_Opr ref) const {
 271   __ testptr(ref->as_register(), address_bad_mask_from_thread(r15_thread));
 272 }
 273 
 274 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
 275                                                          ZLoadBarrierStubC1* stub) const {
 276   // Stub entry
 277   __ bind(*stub->entry());
 278 
 279   Register ref = stub->ref()->as_register();
 280   Register ref_addr = noreg;
 281   Register tmp = noreg;
 282 
 283   if (stub->tmp()->is_valid()) {
 284     // Load address into tmp register
 285     ce->leal(stub->ref_addr(), stub->tmp());
 286     ref_addr = tmp = stub->tmp()->as_pointer_register();
 287   } else {
 288     // Address already in register
 289     ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
 290   }
 291 
 292   assert_different_registers(ref, ref_addr, noreg);
 293 
 294   // Save rax unless it is the result or tmp register
 295   if (ref != rax && tmp != rax) {
 296     __ push(rax);
 297   }
 298 
 299   // Setup arguments and call runtime stub
 300   __ subptr(rsp, 2 * BytesPerWord);
 301   ce->store_parameter(ref_addr, 1);
 302   ce->store_parameter(ref, 0);
 303   __ call(RuntimeAddress(stub->runtime_stub()));
 304   __ addptr(rsp, 2 * BytesPerWord);
 305 
 306   // Verify result
 307   __ verify_oop(rax, "Bad oop");
 308 
 309   // Move result into place
 310   if (ref != rax) {
 311     __ movptr(ref, rax);
 312   }
 313 
 314   // Restore rax unless it is the result or tmp register
 315   if (ref != rax && tmp != rax) {
 316     __ pop(rax);
 317   }
 318 
 319   // Stub exit
 320   __ jmp(*stub->continuation());
 321 }
 322 
 323 #undef __
 324 #define __ sasm->
 325 
 326 void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
 327                                                                  DecoratorSet decorators) const {
 328   // Enter and save registers
 329   __ enter();
 330   __ save_live_registers_no_oop_map(true /* save_fpu_registers */);
 331 
 332   // Setup arguments
 333   __ load_parameter(1, c_rarg1);
 334   __ load_parameter(0, c_rarg0);
 335 
 336   // Call VM
 337   __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
 338 
 339   // Restore registers and return
 340   __ restore_live_registers_except_rax(true /* restore_fpu_registers */);
 341   __ leave();
 342   __ ret(0);
 343 }
 344 
 345 #endif // COMPILER1
 346 
 347 #undef __
 348 #define __ cgen->assembler()->
 349 
 350 // Generates a register specific stub for calling
 351 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
 352 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
 353 //
 354 // The raddr register serves as both input and output for this stub. When the stub is
 355 // called the raddr register contains the object field address (oop*) where the bad oop
 356 // was loaded from, which caused the slow path to be taken. On return from the stub the
 357 // raddr register contains the good/healed oop returned from
 358 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
 359 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
 360 static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) {
 361   // Don't generate stub for invalid registers
 362   if (raddr == rsp || raddr == r12 || raddr == r15) {
 363     return NULL;
 364   }
 365 
 366   // Create stub name
 367   char name[64];
 368   const bool weak = (decorators & ON_WEAK_OOP_REF) != 0;
 369   os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name());
 370 
 371   __ align(CodeEntryAlignment);
 372   StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode));
 373   address start = __ pc();
 374 
 375   // Save live registers
 376   if (raddr != rax) {
 377     __ push(rax);
 378   }
 379   if (raddr != rcx) {
 380     __ push(rcx);
 381   }
 382   if (raddr != rdx) {
 383     __ push(rdx);
 384   }
 385   if (raddr != rsi) {
 386     __ push(rsi);
 387   }
 388   if (raddr != rdi) {
 389     __ push(rdi);
 390   }
 391   if (raddr != r8) {
 392     __ push(r8);
 393   }
 394   if (raddr != r9) {
 395     __ push(r9);
 396   }
 397   if (raddr != r10) {
 398     __ push(r10);
 399   }
 400   if (raddr != r11) {
 401     __ push(r11);
 402   }
 403 
 404   // Setup arguments
 405   if (raddr != c_rarg1) {
 406     __ movq(c_rarg1, raddr);
 407   }
 408   __ movq(c_rarg0, Address(raddr, 0));
 409 
 410   // Call barrier function
 411   __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
 412 
 413   // Move result returned in rax to raddr, if needed
 414   if (raddr != rax) {
 415     __ movq(raddr, rax);
 416   }
 417 
 418   // Restore saved registers
 419   if (raddr != r11) {
 420     __ pop(r11);
 421   }
 422   if (raddr != r10) {
 423     __ pop(r10);
 424   }
 425   if (raddr != r9) {
 426     __ pop(r9);
 427   }
 428   if (raddr != r8) {
 429     __ pop(r8);
 430   }
 431   if (raddr != rdi) {
 432     __ pop(rdi);
 433   }
 434   if (raddr != rsi) {
 435     __ pop(rsi);
 436   }
 437   if (raddr != rdx) {
 438     __ pop(rdx);
 439   }
 440   if (raddr != rcx) {
 441     __ pop(rcx);
 442   }
 443   if (raddr != rax) {
 444     __ pop(rax);
 445   }
 446 
 447   __ ret(0);
 448 
 449   return start;
 450 }
 451 
 452 #undef __
 453 
 454 static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) {
 455   const int nregs = RegisterImpl::number_of_registers;
 456   const int code_size = nregs * 128; // Rough estimate of code size
 457 
 458   ResourceMark rm;
 459 
 460   CodeBuffer buf(BufferBlob::create(label, code_size));
 461   StubCodeGenerator cgen(&buf);
 462 
 463   for (int i = 0; i < nregs; i++) {
 464     const Register reg = as_Register(i);
 465     stub[i] = generate_load_barrier_stub(&cgen, reg, decorators);
 466   }
 467 }
 468 
 469 void ZBarrierSetAssembler::barrier_stubs_init() {
 470   barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub);
 471   barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub);
 472 }
 473 
 474 address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) {
 475   return _load_barrier_slow_stub[reg->encoding()];
 476 }
 477 
 478 address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) {
 479   return _load_barrier_weak_slow_stub[reg->encoding()];
 480 }
--- EOF ---