1 /*
   2  * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/assembler.hpp"
  28 #include "c1/c1_CodeStubs.hpp"
  29 #include "c1/c1_Defs.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "compiler/disassembler.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "nativeInst_aarch64.hpp"
  35 #include "oops/compiledICHolder.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "prims/jvmtiExport.hpp"
  38 #include "register_aarch64.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "runtime/signature.hpp"
  41 #include "runtime/vframe.hpp"
  42 #include "runtime/vframeArray.hpp"
  43 #include "vmreg_aarch64.inline.hpp"
  44 #if INCLUDE_ALL_GCS
  45 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
  46 #endif
  47 
  48 
  49 // Implementation of StubAssembler
  50 
  51 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
  52   // setup registers
  53   assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
  54   assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");
  55   assert(args_size >= 0, "illegal args_size");
  56   bool align_stack = false;
  57 
  58   mov(c_rarg0, rthread);
  59   set_num_rt_args(0); // Nothing on stack
  60 
  61   Label retaddr;
  62   set_last_Java_frame(sp, rfp, retaddr, rscratch1);
  63 
  64   // do the call
  65   lea(rscratch1, RuntimeAddress(entry));
  66   blrt(rscratch1, args_size + 1, 8, 1);
  67   bind(retaddr);
  68   int call_offset = offset();
  69   // verify callee-saved register
  70 #ifdef ASSERT
  71   push(r0, sp);
  72   { Label L;
  73     get_thread(r0);
  74     cmp(rthread, r0);
  75     br(Assembler::EQ, L);
  76     stop("StubAssembler::call_RT: rthread not callee saved?");
  77     bind(L);
  78   }
  79   pop(r0, sp);
  80 #endif
  81   reset_last_Java_frame(true, true);
  82   maybe_isb();
  83 
  84   // check for pending exceptions
  85   { Label L;
  86     // check for pending exceptions (java_thread is set upon return)
  87     ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
  88     cbz(rscratch1, L);
  89     // exception pending => remove activation and forward to exception handler
  90     // make sure that the vm_results are cleared
  91     if (oop_result1->is_valid()) {
  92       str(zr, Address(rthread, JavaThread::vm_result_offset()));
  93     }
  94     if (metadata_result->is_valid()) {
  95       str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
  96     }
  97     if (frame_size() == no_frame_size) {
  98       leave();
  99       far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 100     } else if (_stub_id == Runtime1::forward_exception_id) {
 101       should_not_reach_here();
 102     } else {
 103       far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 104     }
 105     bind(L);
 106   }
 107   // get oop results if there are any and reset the values in the thread
 108   if (oop_result1->is_valid()) {
 109     get_vm_result(oop_result1, rthread);
 110   }
 111   if (metadata_result->is_valid()) {
 112     get_vm_result_2(metadata_result, rthread);
 113   }
 114   return call_offset;
 115 }
 116 
 117 
 118 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
 119   mov(c_rarg1, arg1);
 120   return call_RT(oop_result1, metadata_result, entry, 1);
 121 }
 122 
 123 
 124 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
 125   if (c_rarg1 == arg2) {
 126     if (c_rarg2 == arg1) {
 127       mov(rscratch1, arg1);
 128       mov(arg1, arg2);
 129       mov(arg2, rscratch1);
 130     } else {
 131       mov(c_rarg2, arg2);
 132       mov(c_rarg1, arg1);
 133     }
 134   } else {
 135     mov(c_rarg1, arg1);
 136     mov(c_rarg2, arg2);
 137   }
 138   return call_RT(oop_result1, metadata_result, entry, 2);
 139 }
 140 
 141 
 142 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
 143   // if there is any conflict use the stack
 144   if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
 145       arg2 == c_rarg1 || arg1 == c_rarg3 ||
 146       arg3 == c_rarg1 || arg1 == c_rarg2) {
 147     stp(arg3, arg2, Address(pre(sp, 2 * wordSize)));
 148     stp(arg1, zr, Address(pre(sp, -2 * wordSize)));
 149     ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));
 150     ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));
 151   } else {
 152     mov(c_rarg1, arg1);
 153     mov(c_rarg2, arg2);
 154     mov(c_rarg3, arg3);
 155   }
 156   return call_RT(oop_result1, metadata_result, entry, 3);
 157 }
 158 
 159 // Implementation of StubFrame
 160 
 161 class StubFrame: public StackObj {
 162  private:
 163   StubAssembler* _sasm;
 164 
 165  public:
 166   StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
 167   void load_argument(int offset_in_words, Register reg);
 168 
 169   ~StubFrame();
 170 };;
 171 
 172 
 173 #define __ _sasm->
 174 
 175 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {
 176   _sasm = sasm;
 177   __ set_info(name, must_gc_arguments);
 178   __ enter();
 179 }
 180 
 181 // load parameters that were stored with LIR_Assembler::store_parameter
 182 // Note: offsets for store_parameter and load_argument must match
 183 void StubFrame::load_argument(int offset_in_words, Register reg) {
 184   // rbp, + 0: link
 185   //     + 1: return address
 186   //     + 2: argument with offset 0
 187   //     + 3: argument with offset 1
 188   //     + 4: ...
 189 
 190   __ ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord));
 191 }
 192 
 193 
 194 StubFrame::~StubFrame() {
 195   __ leave();
 196   __ ret(lr);
 197 }
 198 
 199 #undef __
 200 
 201 
 202 // Implementation of Runtime1
 203 
 204 #define __ sasm->
 205 
 206 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
 207 
 208 // Stack layout for saving/restoring  all the registers needed during a runtime
 209 // call (this includes deoptimization)
 210 // Note: note that users of this frame may well have arguments to some runtime
 211 // while these values are on the stack. These positions neglect those arguments
 212 // but the code in save_live_registers will take the argument count into
 213 // account.
 214 //
 215 
 216 enum reg_save_layout {
 217   reg_save_frame_size = 32 /* float */ + 32 /* integer */
 218 };
 219 
 220 // Save off registers which might be killed by calls into the runtime.
 221 // Tries to smart of about FP registers.  In particular we separate
 222 // saving and describing the FPU registers for deoptimization since we
 223 // have to save the FPU registers twice if we describe them.  The
 224 // deopt blob is the only thing which needs to describe FPU registers.
 225 // In all other cases it should be sufficient to simply save their
 226 // current value.
 227 
 228 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
 229 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
 230 static int reg_save_size_in_words;
 231 static int frame_size_in_bytes = -1;
 232 
 233 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
 234   int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
 235   sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
 236   int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
 237   OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
 238 
 239   for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {
 240     Register r = as_Register(i);
 241     if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) {
 242       int sp_offset = cpu_reg_save_offsets[i];
 243       oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 244                                 r->as_VMReg());
 245     }
 246   }
 247 
 248   if (save_fpu_registers) {
 249     for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
 250       FloatRegister r = as_FloatRegister(i);
 251       {
 252         int sp_offset = fpu_reg_save_offsets[i];
 253         oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 254                                   r->as_VMReg());
 255       }
 256     }
 257   }
 258   return oop_map;
 259 }
 260 
 261 static OopMap* save_live_registers(StubAssembler* sasm,
 262                                    bool save_fpu_registers = true) {
 263   __ block_comment("save_live_registers");
 264 
 265   __ push(RegSet::range(r0, r29), sp);         // integer registers except lr & sp
 266 
 267   if (save_fpu_registers) {
 268     for (int i = 30; i >= 0; i -= 2)
 269       __ stpd(as_FloatRegister(i), as_FloatRegister(i+1),
 270               Address(__ pre(sp, -2 * wordSize)));
 271   } else {
 272     __ add(sp, sp, -32 * wordSize);
 273   }
 274 
 275   return generate_oop_map(sasm, save_fpu_registers);
 276 }
 277 
 278 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
 279   if (restore_fpu_registers) {
 280     for (int i = 0; i < 32; i += 2)
 281       __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
 282               Address(__ post(sp, 2 * wordSize)));
 283   } else {
 284     __ add(sp, sp, 32 * wordSize);
 285   }
 286 
 287   __ pop(RegSet::range(r0, r29), sp);
 288 }
 289 
 290 static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true)  {
 291 
 292   if (restore_fpu_registers) {
 293     for (int i = 0; i < 32; i += 2)
 294       __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
 295               Address(__ post(sp, 2 * wordSize)));
 296   } else {
 297     __ add(sp, sp, 32 * wordSize);
 298   }
 299 
 300   __ ldp(zr, r1, Address(__ post(sp, 16)));
 301   __ pop(RegSet::range(r2, r29), sp);
 302 }
 303 
 304 
 305 
 306 void Runtime1::initialize_pd() {
 307   int i;
 308   int sp_offset = 0;
 309 
 310   // all float registers are saved explicitly
 311   assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");
 312   for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
 313     fpu_reg_save_offsets[i] = sp_offset;
 314     sp_offset += 2;   // SP offsets are in halfwords
 315   }
 316 
 317   for (i = 0; i < FrameMap::nof_cpu_regs; i++) {
 318     Register r = as_Register(i);
 319     cpu_reg_save_offsets[i] = sp_offset;
 320     sp_offset += 2;   // SP offsets are in halfwords
 321   }
 322 }
 323 
 324 
 325 // target: the entry point of the method that creates and posts the exception oop
 326 // has_argument: true if the exception needs an argument (passed in rscratch1)
 327 
 328 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
 329   // make a frame and preserve the caller's caller-save registers
 330   OopMap* oop_map = save_live_registers(sasm);
 331   int call_offset;
 332   if (!has_argument) {
 333     call_offset = __ call_RT(noreg, noreg, target);
 334   } else {
 335     call_offset = __ call_RT(noreg, noreg, target, rscratch1);
 336   }
 337   OopMapSet* oop_maps = new OopMapSet();
 338   oop_maps->add_gc_map(call_offset, oop_map);
 339 
 340   __ should_not_reach_here();
 341   return oop_maps;
 342 }
 343 
 344 
 345 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
 346   __ block_comment("generate_handle_exception");
 347 
 348   // incoming parameters
 349   const Register exception_oop = r0;
 350   const Register exception_pc  = r3;
 351   // other registers used in this stub
 352 
 353   // Save registers, if required.
 354   OopMapSet* oop_maps = new OopMapSet();
 355   OopMap* oop_map = NULL;
 356   switch (id) {
 357   case forward_exception_id:
 358     // We're handling an exception in the context of a compiled frame.
 359     // The registers have been saved in the standard places.  Perform
 360     // an exception lookup in the caller and dispatch to the handler
 361     // if found.  Otherwise unwind and dispatch to the callers
 362     // exception handler.
 363     oop_map = generate_oop_map(sasm, 1 /*thread*/);
 364 
 365     // load and clear pending exception oop into r0
 366     __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));
 367     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 368 
 369     // load issuing PC (the return address for this stub) into r3
 370     __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
 371 
 372     // make sure that the vm_results are cleared (may be unnecessary)
 373     __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
 374     __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
 375     break;
 376   case handle_exception_nofpu_id:
 377   case handle_exception_id:
 378     // At this point all registers MAY be live.
 379     oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
 380     break;
 381   case handle_exception_from_callee_id: {
 382     // At this point all registers except exception oop (r0) and
 383     // exception pc (lr) are dead.
 384     const int frame_size = 2 /*fp, return address*/;
 385     oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
 386     sasm->set_frame_size(frame_size);
 387     break;
 388   }
 389   default:
 390     __ should_not_reach_here();
 391     break;
 392   }
 393 
 394   // verify that only r0 and r3 are valid at this time
 395   __ invalidate_registers(false, true, true, false, true, true);
 396   // verify that r0 contains a valid exception
 397   __ verify_not_null_oop(exception_oop);
 398 
 399 #ifdef ASSERT
 400   // check that fields in JavaThread for exception oop and issuing pc are
 401   // empty before writing to them
 402   Label oop_empty;
 403   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 404   __ cbz(rscratch1, oop_empty);
 405   __ stop("exception oop already set");
 406   __ bind(oop_empty);
 407 
 408   Label pc_empty;
 409   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 410   __ cbz(rscratch1, pc_empty);
 411   __ stop("exception pc already set");
 412   __ bind(pc_empty);
 413 #endif
 414 
 415   // save exception oop and issuing pc into JavaThread
 416   // (exception handler will load it from here)
 417   __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));
 418   __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
 419 
 420   // patch throwing pc into return address (has bci & oop map)
 421   __ str(exception_pc, Address(rfp, 1*BytesPerWord));
 422 
 423   // compute the exception handler.
 424   // the exception oop and the throwing pc are read from the fields in JavaThread
 425   int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
 426   oop_maps->add_gc_map(call_offset, oop_map);
 427 
 428   // r0: handler address
 429   //      will be the deopt blob if nmethod was deoptimized while we looked up
 430   //      handler regardless of whether handler existed in the nmethod.
 431 
 432   // only r0 is valid at this time, all other registers have been destroyed by the runtime call
 433   __ invalidate_registers(false, true, true, true, true, true);
 434 
 435   // patch the return address, this stub will directly return to the exception handler
 436   __ str(r0, Address(rfp, 1*BytesPerWord));
 437 
 438   switch (id) {
 439   case forward_exception_id:
 440   case handle_exception_nofpu_id:
 441   case handle_exception_id:
 442     // Restore the registers that were saved at the beginning.
 443     restore_live_registers(sasm, id != handle_exception_nofpu_id);
 444     break;
 445   case handle_exception_from_callee_id:
 446     // Pop the return address.
 447     __ leave();
 448     __ ret(lr);  // jump to exception handler
 449     break;
 450   default:  ShouldNotReachHere();
 451   }
 452 
 453   return oop_maps;
 454 }
 455 
 456 
 457 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
 458   // incoming parameters
 459   const Register exception_oop = r0;
 460   // callee-saved copy of exception_oop during runtime call
 461   const Register exception_oop_callee_saved = r19;
 462   // other registers used in this stub
 463   const Register exception_pc = r3;
 464   const Register handler_addr = r1;
 465 
 466   // verify that only r0, is valid at this time
 467   __ invalidate_registers(false, true, true, true, true, true);
 468 
 469 #ifdef ASSERT
 470   // check that fields in JavaThread for exception oop and issuing pc are empty
 471   Label oop_empty;
 472   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 473   __ cbz(rscratch1, oop_empty);
 474   __ stop("exception oop must be empty");
 475   __ bind(oop_empty);
 476 
 477   Label pc_empty;
 478   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 479   __ cbz(rscratch1, pc_empty);
 480   __ stop("exception pc must be empty");
 481   __ bind(pc_empty);
 482 #endif
 483 
 484   // Save our return address because
 485   // exception_handler_for_return_address will destroy it.  We also
 486   // save exception_oop
 487   __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
 488 
 489   // search the exception handler address of the caller (using the return address)
 490   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr);
 491   // r0: exception handler address of the caller
 492 
 493   // Only R0 is valid at this time; all other registers have been
 494   // destroyed by the call.
 495   __ invalidate_registers(false, true, true, true, false, true);
 496 
 497   // move result of call into correct register
 498   __ mov(handler_addr, r0);
 499 
 500   // get throwing pc (= return address).
 501   // lr has been destroyed by the call
 502   __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
 503   __ mov(r3, lr);
 504 
 505   __ verify_not_null_oop(exception_oop);
 506 
 507   // continue at exception handler (return address removed)
 508   // note: do *not* remove arguments when unwinding the
 509   //       activation since the caller assumes having
 510   //       all arguments on the stack when entering the
 511   //       runtime to determine the exception handler
 512   //       (GC happens at call site with arguments!)
 513   // r0: exception oop
 514   // r3: throwing pc
 515   // r1: exception handler
 516   __ br(handler_addr);
 517 }
 518 
 519 
 520 
 521 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
 522   // use the maximum number of runtime-arguments here because it is difficult to
 523   // distinguish each RT-Call.
 524   // Note: This number affects also the RT-Call in generate_handle_exception because
 525   //       the oop-map is shared for all calls.
 526   DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
 527   assert(deopt_blob != NULL, "deoptimization blob must have been created");
 528 
 529   OopMap* oop_map = save_live_registers(sasm);
 530 
 531   __ mov(c_rarg0, rthread);
 532   Label retaddr;
 533   __ set_last_Java_frame(sp, rfp, retaddr, rscratch1);
 534   // do the call
 535   __ lea(rscratch1, RuntimeAddress(target));
 536   __ blrt(rscratch1, 1, 0, 1);
 537   __ bind(retaddr);
 538   OopMapSet* oop_maps = new OopMapSet();
 539   oop_maps->add_gc_map(__ offset(), oop_map);
 540   // verify callee-saved register
 541 #ifdef ASSERT
 542   { Label L;
 543     __ get_thread(rscratch1);
 544     __ cmp(rthread, rscratch1);
 545     __ br(Assembler::EQ, L);
 546     __ stop("StubAssembler::call_RT: rthread not callee saved?");
 547     __ bind(L);
 548   }
 549 #endif
 550   __ reset_last_Java_frame(true, false);
 551   __ maybe_isb();
 552 
 553   // check for pending exceptions
 554   { Label L;
 555     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 556     __ cbz(rscratch1, L);
 557     // exception pending => remove activation and forward to exception handler
 558 
 559     { Label L1;
 560       __ cbnz(r0, L1);                                  // have we deoptimized?
 561       __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 562       __ bind(L1);
 563     }
 564 
 565     // the deopt blob expects exceptions in the special fields of
 566     // JavaThread, so copy and clear pending exception.
 567 
 568     // load and clear pending exception
 569     __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 570     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 571 
 572     // check that there is really a valid exception
 573     __ verify_not_null_oop(r0);
 574 
 575     // load throwing pc: this is the return address of the stub
 576     __ mov(r3, lr);
 577 
 578 #ifdef ASSERT
 579     // check that fields in JavaThread for exception oop and issuing pc are empty
 580     Label oop_empty;
 581     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 582     __ cbz(rscratch1, oop_empty);
 583     __ stop("exception oop must be empty");
 584     __ bind(oop_empty);
 585 
 586     Label pc_empty;
 587     __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 588     __ cbz(rscratch1, pc_empty);
 589     __ stop("exception pc must be empty");
 590     __ bind(pc_empty);
 591 #endif
 592 
 593     // store exception oop and throwing pc to JavaThread
 594     __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
 595     __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
 596 
 597     restore_live_registers(sasm);
 598 
 599     __ leave();
 600 
 601     // Forward the exception directly to deopt blob. We can blow no
 602     // registers and must leave throwing pc on the stack.  A patch may
 603     // have values live in registers so the entry point with the
 604     // exception in tls.
 605     __ far_jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls()));
 606 
 607     __ bind(L);
 608   }
 609 
 610 
 611   // Runtime will return true if the nmethod has been deoptimized during
 612   // the patching process. In that case we must do a deopt reexecute instead.
 613 
 614   Label reexecuteEntry, cont;
 615 
 616   __ cbz(r0, cont);                                 // have we deoptimized?
 617 
 618   // Will reexecute. Proper return address is already on the stack we just restore
 619   // registers, pop all of our frame but the return address and jump to the deopt blob
 620   restore_live_registers(sasm);
 621   __ leave();
 622   __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
 623 
 624   __ bind(cont);
 625   restore_live_registers(sasm);
 626   __ leave();
 627   __ ret(lr);
 628 
 629   return oop_maps;
 630 }
 631 
 632 
 633 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
 634 
 635   const Register exception_oop = r0;
 636   const Register exception_pc  = r3;
 637 
 638   // for better readability
 639   const bool must_gc_arguments = true;
 640   const bool dont_gc_arguments = false;
 641 
 642   // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
 643   bool save_fpu_registers = true;
 644 
 645   // stub code & info for the different stubs
 646   OopMapSet* oop_maps = NULL;
 647   OopMap* oop_map = NULL;
 648   switch (id) {
 649     {
 650     case forward_exception_id:
 651       {
 652         oop_maps = generate_handle_exception(id, sasm);
 653         __ leave();
 654         __ ret(lr);
 655       }
 656       break;
 657 
 658     case throw_div0_exception_id:
 659       { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments);
 660         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
 661       }
 662       break;
 663 
 664     case throw_null_pointer_exception_id:
 665       { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments);
 666         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
 667       }
 668       break;
 669 
 670     case new_instance_id:
 671     case fast_new_instance_id:
 672     case fast_new_instance_init_check_id:
 673       {
 674         Register klass = r3; // Incoming
 675         Register obj   = r0; // Result
 676 
 677         if (id == new_instance_id) {
 678           __ set_info("new_instance", dont_gc_arguments);
 679         } else if (id == fast_new_instance_id) {
 680           __ set_info("fast new_instance", dont_gc_arguments);
 681         } else {
 682           assert(id == fast_new_instance_init_check_id, "bad StubID");
 683           __ set_info("fast new_instance init check", dont_gc_arguments);
 684         }
 685 
 686         if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
 687             UseTLAB && FastTLABRefill) {
 688           Label slow_path;
 689           Register obj_size = r2;
 690           Register t1       = r19;
 691           Register t2       = r4;
 692           assert_different_registers(klass, obj, obj_size, t1, t2);
 693 
 694           __ stp(r5, r19, Address(__ pre(sp, -2 * wordSize)));
 695 
 696           if (id == fast_new_instance_init_check_id) {
 697             // make sure the klass is initialized
 698             __ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset()));
 699             __ cmpw(rscratch1, InstanceKlass::fully_initialized);
 700             __ br(Assembler::NE, slow_path);
 701           }
 702 
 703 #ifdef ASSERT
 704           // assert object can be fast path allocated
 705           {
 706             Label ok, not_ok;
 707             __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 708             __ cmp(obj_size, 0u);
 709             __ br(Assembler::LE, not_ok);  // make sure it's an instance (LH > 0)
 710             __ tstw(obj_size, Klass::_lh_instance_slow_path_bit);
 711             __ br(Assembler::EQ, ok);
 712             __ bind(not_ok);
 713             __ stop("assert(can be fast path allocated)");
 714             __ should_not_reach_here();
 715             __ bind(ok);
 716           }
 717 #endif // ASSERT
 718 
 719           // if we got here then the TLAB allocation failed, so try
 720           // refilling the TLAB or allocating directly from eden.
 721           Label retry_tlab, try_eden;
 722           __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy r3 (klass), returns r5
 723 
 724           __ bind(retry_tlab);
 725 
 726           // get the instance size (size is postive so movl is fine for 64bit)
 727           __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 728 
 729           __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path);
 730 
 731           __ initialize_object(obj, klass, obj_size, 0, t1, t2);
 732           __ verify_oop(obj);
 733           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 734           __ ret(lr);
 735 
 736           __ bind(try_eden);
 737           // get the instance size (size is postive so movl is fine for 64bit)
 738           __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 739 
 740           __ eden_allocate(obj, obj_size, 0, t1, slow_path);
 741           __ incr_allocated_bytes(rthread, obj_size, 0, rscratch1);
 742 
 743           __ initialize_object(obj, klass, obj_size, 0, t1, t2);
 744           __ verify_oop(obj);
 745           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 746           __ ret(lr);
 747 
 748           __ bind(slow_path);
 749           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 750         }
 751 
 752         __ enter();
 753         OopMap* map = save_live_registers(sasm);
 754         int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
 755         oop_maps = new OopMapSet();
 756         oop_maps->add_gc_map(call_offset, map);
 757         restore_live_registers_except_r0(sasm);
 758         __ verify_oop(obj);
 759         __ leave();
 760         __ ret(lr);
 761 
 762         // r0,: new instance
 763       }
 764 
 765       break;
 766 
 767     case counter_overflow_id:
 768       {
 769         Register bci = r0, method = r1;
 770         __ enter();
 771         OopMap* map = save_live_registers(sasm);
 772         // Retrieve bci
 773         __ ldrw(bci, Address(rfp, 2*BytesPerWord));
 774         // And a pointer to the Method*
 775         __ ldr(method, Address(rfp, 3*BytesPerWord));
 776         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
 777         oop_maps = new OopMapSet();
 778         oop_maps->add_gc_map(call_offset, map);
 779         restore_live_registers(sasm);
 780         __ leave();
 781         __ ret(lr);
 782       }
 783       break;
 784 
 785     case new_type_array_id:
 786     case new_object_array_id:
 787       {
 788         Register length   = r19; // Incoming
 789         Register klass    = r3; // Incoming
 790         Register obj      = r0; // Result
 791 
 792         if (id == new_type_array_id) {
 793           __ set_info("new_type_array", dont_gc_arguments);
 794         } else {
 795           __ set_info("new_object_array", dont_gc_arguments);
 796         }
 797 
 798 #ifdef ASSERT
 799         // assert object type is really an array of the proper kind
 800         {
 801           Label ok;
 802           Register t0 = obj;
 803           __ ldrw(t0, Address(klass, Klass::layout_helper_offset()));
 804           __ asrw(t0, t0, Klass::_lh_array_tag_shift);
 805           int tag = ((id == new_type_array_id)
 806                      ? Klass::_lh_array_tag_type_value
 807                      : Klass::_lh_array_tag_obj_value);
 808           __ mov(rscratch1, tag);
 809           __ cmpw(t0, rscratch1);
 810           __ br(Assembler::EQ, ok);
 811           __ stop("assert(is an array klass)");
 812           __ should_not_reach_here();
 813           __ bind(ok);
 814         }
 815 #endif // ASSERT
 816 
 817         if (UseTLAB && FastTLABRefill) {
 818           Register arr_size = r4;
 819           Register t1       = r2;
 820           Register t2       = r5;
 821           Label slow_path;
 822           assert_different_registers(length, klass, obj, arr_size, t1, t2);
 823 
 824           // check that array length is small enough for fast path.
 825           __ mov(rscratch1, C1_MacroAssembler::max_array_allocation_length);
 826           __ cmpw(length, rscratch1);
 827           __ br(Assembler::HI, slow_path);
 828 
 829           // if we got here then the TLAB allocation failed, so try
 830           // refilling the TLAB or allocating directly from eden.
 831           Label retry_tlab, try_eden;
 832           const Register thread =
 833             __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves r19 & r3, returns rthread
 834 
 835           __ bind(retry_tlab);
 836 
 837           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
 838           // since size is positive ldrw does right thing on 64bit
 839           __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
 840           __ lslvw(arr_size, length, t1);
 841           __ ubfx(t1, t1, Klass::_lh_header_size_shift,
 842                   exact_log2(Klass::_lh_header_size_mask + 1));
 843           __ add(arr_size, arr_size, t1);
 844           __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
 845           __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 846 
 847           __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path);  // preserves arr_size
 848 
 849           __ initialize_header(obj, klass, length, t1, t2);
 850           __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
 851           assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
 852           assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
 853           __ andr(t1, t1, Klass::_lh_header_size_mask);
 854           __ sub(arr_size, arr_size, t1);  // body length
 855           __ add(t1, t1, obj);       // body start
 856           __ initialize_body(t1, arr_size, 0, t2);
 857           __ verify_oop(obj);
 858 
 859           __ ret(lr);
 860 
 861           __ bind(try_eden);
 862           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
 863           // since size is positive ldrw does right thing on 64bit
 864           __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
 865           // since size is postive movw does right thing on 64bit
 866           __ movw(arr_size, length);
 867           __ lslvw(arr_size, length, t1);
 868           __ ubfx(t1, t1, Klass::_lh_header_size_shift,
 869                   exact_log2(Klass::_lh_header_size_mask + 1));
 870           __ add(arr_size, arr_size, t1);
 871           __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
 872           __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 873 
 874           __ eden_allocate(obj, arr_size, 0, t1, slow_path);  // preserves arr_size
 875           __ incr_allocated_bytes(thread, arr_size, 0, rscratch1);
 876 
 877           __ initialize_header(obj, klass, length, t1, t2);
 878           __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
 879           assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
 880           assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
 881           __ andr(t1, t1, Klass::_lh_header_size_mask);
 882           __ sub(arr_size, arr_size, t1);  // body length
 883           __ add(t1, t1, obj);       // body start
 884           __ initialize_body(t1, arr_size, 0, t2);
 885           __ verify_oop(obj);
 886 
 887           __ ret(lr);
 888 
 889           __ bind(slow_path);
 890         }
 891 
 892         __ enter();
 893         OopMap* map = save_live_registers(sasm);
 894         int call_offset;
 895         if (id == new_type_array_id) {
 896           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
 897         } else {
 898           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
 899         }
 900 
 901         oop_maps = new OopMapSet();
 902         oop_maps->add_gc_map(call_offset, map);
 903         restore_live_registers_except_r0(sasm);
 904 
 905         __ verify_oop(obj);
 906         __ leave();
 907         __ ret(lr);
 908 
 909         // r0: new array
 910       }
 911       break;
 912 
 913     case new_multi_array_id:
 914       { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
 915         // r0,: klass
 916         // r19,: rank
 917         // r2: address of 1st dimension
 918         OopMap* map = save_live_registers(sasm);
 919         __ mov(c_rarg1, r0);
 920         __ mov(c_rarg3, r2);
 921         __ mov(c_rarg2, r19);
 922         int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3);
 923 
 924         oop_maps = new OopMapSet();
 925         oop_maps->add_gc_map(call_offset, map);
 926         restore_live_registers_except_r0(sasm);
 927 
 928         // r0,: new multi array
 929         __ verify_oop(r0);
 930       }
 931       break;
 932 
 933     case register_finalizer_id:
 934       {
 935         __ set_info("register_finalizer", dont_gc_arguments);
 936 
 937         // This is called via call_runtime so the arguments
 938         // will be place in C abi locations
 939 
 940         __ verify_oop(c_rarg0);
 941 
 942         // load the klass and check the has finalizer flag
 943         Label register_finalizer;
 944         Register t = r5;
 945         __ load_klass(t, r0);
 946         __ ldrw(t, Address(t, Klass::access_flags_offset()));
 947         __ tst(t, JVM_ACC_HAS_FINALIZER);
 948         __ br(Assembler::NE, register_finalizer);
 949         __ ret(lr);
 950 
 951         __ bind(register_finalizer);
 952         __ enter();
 953         OopMap* oop_map = save_live_registers(sasm);
 954         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0);
 955         oop_maps = new OopMapSet();
 956         oop_maps->add_gc_map(call_offset, oop_map);
 957 
 958         // Now restore all the live registers
 959         restore_live_registers(sasm);
 960 
 961         __ leave();
 962         __ ret(lr);
 963       }
 964       break;
 965 
 966     case throw_class_cast_exception_id:
 967       { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments);
 968         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
 969       }
 970       break;
 971 
 972     case throw_incompatible_class_change_error_id:
 973       { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments);
 974         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
 975       }
 976       break;
 977 
 978     case slow_subtype_check_id:
 979       {
 980         // Typical calling sequence:
 981         // __ push(klass_RInfo);  // object klass or other subclass
 982         // __ push(sup_k_RInfo);  // array element klass or other superclass
 983         // __ bl(slow_subtype_check);
 984         // Note that the subclass is pushed first, and is therefore deepest.
 985         enum layout {
 986           r0_off, r0_off_hi,
 987           r2_off, r2_off_hi,
 988           r4_off, r4_off_hi,
 989           r5_off, r5_off_hi,
 990           sup_k_off, sup_k_off_hi,
 991           klass_off, klass_off_hi,
 992           framesize,
 993           result_off = sup_k_off
 994         };
 995 
 996         __ set_info("slow_subtype_check", dont_gc_arguments);
 997         __ push(RegSet::of(r0, r2, r4, r5), sp);
 998 
 999         // This is called by pushing args and not with C abi
1000         // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
1001         // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
1002 
1003         __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
1004 
1005         Label miss;
1006         __ check_klass_subtype_slow_path(r4, r0, r2, r5, NULL, &miss);
1007 
1008         // fallthrough on success:
1009         __ mov(rscratch1, 1);
1010         __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
1011         __ pop(RegSet::of(r0, r2, r4, r5), sp);
1012         __ ret(lr);
1013 
1014         __ bind(miss);
1015         __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
1016         __ pop(RegSet::of(r0, r2, r4, r5), sp);
1017         __ ret(lr);
1018       }
1019       break;
1020 
1021     case monitorenter_nofpu_id:
1022       save_fpu_registers = false;
1023       // fall through
1024     case monitorenter_id:
1025       {
1026         StubFrame f(sasm, "monitorenter", dont_gc_arguments);
1027         OopMap* map = save_live_registers(sasm, save_fpu_registers);
1028 
1029         // Called with store_parameter and not C abi
1030 
1031         f.load_argument(1, r0); // r0,: object
1032         f.load_argument(0, r1); // r1,: lock address
1033 
1034         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1);
1035 
1036         oop_maps = new OopMapSet();
1037         oop_maps->add_gc_map(call_offset, map);
1038         restore_live_registers(sasm, save_fpu_registers);
1039       }
1040       break;
1041 
1042     case monitorexit_nofpu_id:
1043       save_fpu_registers = false;
1044       // fall through
1045     case monitorexit_id:
1046       {
1047         StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1048         OopMap* map = save_live_registers(sasm, save_fpu_registers);
1049 
1050         // Called with store_parameter and not C abi
1051 
1052         f.load_argument(0, r0); // r0,: lock address
1053 
1054         // note: really a leaf routine but must setup last java sp
1055         //       => use call_RT for now (speed can be improved by
1056         //       doing last java sp setup manually)
1057         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0);
1058 
1059         oop_maps = new OopMapSet();
1060         oop_maps->add_gc_map(call_offset, map);
1061         restore_live_registers(sasm, save_fpu_registers);
1062       }
1063       break;
1064 
1065     case deoptimize_id:
1066       {
1067         StubFrame f(sasm, "deoptimize", dont_gc_arguments);
1068         OopMap* oop_map = save_live_registers(sasm);
1069         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize));
1070         oop_maps = new OopMapSet();
1071         oop_maps->add_gc_map(call_offset, oop_map);
1072         restore_live_registers(sasm);
1073         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1074         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1075         __ leave();
1076         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1077       }
1078       break;
1079 
1080     case throw_range_check_failed_id:
1081       { StubFrame f(sasm, "range_check_failed", dont_gc_arguments);
1082         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1083       }
1084       break;
1085 
1086     case unwind_exception_id:
1087       { __ set_info("unwind_exception", dont_gc_arguments);
1088         // note: no stubframe since we are about to leave the current
1089         //       activation and we are calling a leaf VM function only.
1090         generate_unwind_exception(sasm);
1091       }
1092       break;
1093 
1094     case access_field_patching_id:
1095       { StubFrame f(sasm, "access_field_patching", dont_gc_arguments);
1096         // we should set up register map
1097         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1098       }
1099       break;
1100 
1101     case load_klass_patching_id:
1102       { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments);
1103         // we should set up register map
1104         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1105       }
1106       break;
1107 
1108     case load_mirror_patching_id:
1109       { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments);
1110         // we should set up register map
1111         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1112       }
1113       break;
1114 
1115     case load_appendix_patching_id:
1116       { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments);
1117         // we should set up register map
1118         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1119       }
1120       break;
1121 
1122     case handle_exception_nofpu_id:
1123     case handle_exception_id:
1124       { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1125         oop_maps = generate_handle_exception(id, sasm);
1126       }
1127       break;
1128 
1129     case handle_exception_from_callee_id:
1130       { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1131         oop_maps = generate_handle_exception(id, sasm);
1132       }
1133       break;
1134 
1135     case throw_index_exception_id:
1136       { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments);
1137         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1138       }
1139       break;
1140 
1141     case throw_array_store_exception_id:
1142       { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments);
1143         // tos + 0: link
1144         //     + 1: return address
1145         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1146       }
1147       break;
1148 
1149 #if INCLUDE_ALL_GCS
1150 
1151 // Registers to be saved around calls to g1_wb_pre or g1_wb_post
1152 #define G1_SAVE_REGS (RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2))
1153 
1154     case g1_pre_barrier_slow_id:
1155       {
1156         StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments);
1157         // arg0 : previous value of memory
1158 
1159         BarrierSet* bs = Universe::heap()->barrier_set();
1160         if (bs->kind() != BarrierSet::G1SATBCTLogging) {
1161           __ mov(r0, (int)id);
1162           __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1163           __ should_not_reach_here();
1164           break;
1165         }
1166 
1167         const Register pre_val = r0;
1168         const Register thread = rthread;
1169         const Register tmp = rscratch1;
1170 
1171         Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1172                                              PtrQueue::byte_offset_of_active()));
1173 
1174         Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1175                                              PtrQueue::byte_offset_of_index()));
1176         Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1177                                         PtrQueue::byte_offset_of_buf()));
1178 
1179         Label done;
1180         Label runtime;
1181 
1182         // Can we store original value in the thread's buffer?
1183         __ ldr(tmp, queue_index);
1184         __ cbz(tmp, runtime);
1185 
1186         __ sub(tmp, tmp, wordSize);
1187         __ str(tmp, queue_index);
1188         __ ldr(rscratch2, buffer);
1189         __ add(tmp, tmp, rscratch2);
1190         f.load_argument(0, rscratch2);
1191         __ str(rscratch2, Address(tmp, 0));
1192         __ b(done);
1193 
1194         __ bind(runtime);
1195         __ push(G1_SAVE_REGS, sp);
1196         f.load_argument(0, pre_val);
1197         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
1198         __ pop(G1_SAVE_REGS, sp);
1199         __ bind(done);
1200       }
1201       break;
1202     case g1_post_barrier_slow_id:
1203       {
1204         StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments);
1205 
1206         // arg0: store_address
1207         Address store_addr(rfp, 2*BytesPerWord);
1208 
1209         BarrierSet* bs = Universe::heap()->barrier_set();
1210         CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1211         assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1212 
1213         Label done;
1214         Label runtime;
1215 
1216         // At this point we know new_value is non-NULL and the new_value crosses regions.
1217         // Must check to see if card is already dirty
1218 
1219         const Register thread = rthread;
1220 
1221         Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1222                                              PtrQueue::byte_offset_of_index()));
1223         Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1224                                         PtrQueue::byte_offset_of_buf()));
1225 
1226         const Register card_addr = rscratch2;
1227         ExternalAddress cardtable((address) ct->byte_map_base);
1228 
1229         f.load_argument(0, card_addr);
1230         __ lsr(card_addr, card_addr, CardTableModRefBS::card_shift);
1231         unsigned long offset;
1232         __ adrp(rscratch1, cardtable, offset);
1233         __ add(card_addr, card_addr, rscratch1);
1234         __ ldrb(rscratch1, Address(card_addr, offset));
1235         __ cmpw(rscratch1, (int)G1SATBCardTableModRefBS::g1_young_card_val());
1236         __ br(Assembler::EQ, done);
1237 
1238         assert((int)CardTableModRefBS::dirty_card_val() == 0, "must be 0");
1239 
1240         __ membar(Assembler::StoreLoad);
1241         __ ldrb(rscratch1, Address(card_addr, offset));
1242         __ cbzw(rscratch1, done);
1243 
1244         // storing region crossing non-NULL, card is clean.
1245         // dirty card and log.
1246         __ strb(zr, Address(card_addr, offset));
1247 
1248         __ ldr(rscratch1, queue_index);
1249         __ cbz(rscratch1, runtime);
1250         __ sub(rscratch1, rscratch1, wordSize);
1251         __ str(rscratch1, queue_index);
1252 
1253         const Register buffer_addr = r0;
1254 
1255         __ push(RegSet::of(r0, r1), sp);
1256         __ ldr(buffer_addr, buffer);
1257         __ str(card_addr, Address(buffer_addr, rscratch1));
1258         __ pop(RegSet::of(r0, r1), sp);
1259         __ b(done);
1260 
1261         __ bind(runtime);
1262         __ push(G1_SAVE_REGS, sp);
1263         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
1264         __ pop(G1_SAVE_REGS, sp);
1265         __ bind(done);
1266 
1267       }
1268       break;
1269 #endif
1270 
1271     case predicate_failed_trap_id:
1272       {
1273         StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1274 
1275         OopMap* map = save_live_registers(sasm);
1276 
1277         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1278         oop_maps = new OopMapSet();
1279         oop_maps->add_gc_map(call_offset, map);
1280         restore_live_registers(sasm);
1281         __ leave();
1282         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1283         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1284 
1285         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1286       }
1287       break;
1288 
1289 
1290     default:
1291       { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
1292         __ mov(r0, (int)id);
1293         __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1294         __ should_not_reach_here();
1295       }
1296       break;
1297     }
1298   }
1299   return oop_maps;
1300 }
1301 
1302 #undef __
1303 
1304 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }