rev 49827 : 8201593: Print array length in ArrayIndexOutOfBoundsException.

   1 /*
   2  * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/assembler.hpp"
  28 #include "c1/c1_CodeStubs.hpp"
  29 #include "c1/c1_Defs.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "compiler/disassembler.hpp"
  33 #include "gc/shared/cardTable.hpp"
  34 #include "gc/shared/cardTableBarrierSet.hpp"
  35 #include "interpreter/interpreter.hpp"
  36 #include "nativeInst_aarch64.hpp"
  37 #include "oops/compiledICHolder.hpp"
  38 #include "oops/oop.inline.hpp"
  39 #include "prims/jvmtiExport.hpp"
  40 #include "register_aarch64.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 #include "runtime/signature.hpp"
  43 #include "runtime/vframe.hpp"
  44 #include "runtime/vframeArray.hpp"
  45 #include "vmreg_aarch64.inline.hpp"
  46 #if INCLUDE_ALL_GCS
  47 #include "gc/g1/g1BarrierSet.hpp"
  48 #include "gc/g1/g1CardTable.hpp"
  49 #include "gc/g1/g1ThreadLocalData.hpp"
  50 #endif
  51 
  52 
  53 // Implementation of StubAssembler
  54 
  55 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
  56   // setup registers
  57   assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
  58   assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");
  59   assert(args_size >= 0, "illegal args_size");
  60   bool align_stack = false;
  61 
  62   mov(c_rarg0, rthread);
  63   set_num_rt_args(0); // Nothing on stack
  64 
  65   Label retaddr;
  66   set_last_Java_frame(sp, rfp, retaddr, rscratch1);
  67 
  68   // do the call
  69   lea(rscratch1, RuntimeAddress(entry));
  70   blrt(rscratch1, args_size + 1, 8, 1);
  71   bind(retaddr);
  72   int call_offset = offset();
  73   // verify callee-saved register
  74 #ifdef ASSERT
  75   push(r0, sp);
  76   { Label L;
  77     get_thread(r0);
  78     cmp(rthread, r0);
  79     br(Assembler::EQ, L);
  80     stop("StubAssembler::call_RT: rthread not callee saved?");
  81     bind(L);
  82   }
  83   pop(r0, sp);
  84 #endif
  85   reset_last_Java_frame(true);
  86   maybe_isb();
  87 
  88   // check for pending exceptions
  89   { Label L;
  90     // check for pending exceptions (java_thread is set upon return)
  91     ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
  92     cbz(rscratch1, L);
  93     // exception pending => remove activation and forward to exception handler
  94     // make sure that the vm_results are cleared
  95     if (oop_result1->is_valid()) {
  96       str(zr, Address(rthread, JavaThread::vm_result_offset()));
  97     }
  98     if (metadata_result->is_valid()) {
  99       str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
 100     }
 101     if (frame_size() == no_frame_size) {
 102       leave();
 103       far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 104     } else if (_stub_id == Runtime1::forward_exception_id) {
 105       should_not_reach_here();
 106     } else {
 107       far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 108     }
 109     bind(L);
 110   }
 111   // get oop results if there are any and reset the values in the thread
 112   if (oop_result1->is_valid()) {
 113     get_vm_result(oop_result1, rthread);
 114   }
 115   if (metadata_result->is_valid()) {
 116     get_vm_result_2(metadata_result, rthread);
 117   }
 118   return call_offset;
 119 }
 120 
 121 
 122 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
 123   mov(c_rarg1, arg1);
 124   return call_RT(oop_result1, metadata_result, entry, 1);
 125 }
 126 
 127 
 128 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
 129   if (c_rarg1 == arg2) {
 130     if (c_rarg2 == arg1) {
 131       mov(rscratch1, arg1);
 132       mov(arg1, arg2);
 133       mov(arg2, rscratch1);
 134     } else {
 135       mov(c_rarg2, arg2);
 136       mov(c_rarg1, arg1);
 137     }
 138   } else {
 139     mov(c_rarg1, arg1);
 140     mov(c_rarg2, arg2);
 141   }
 142   return call_RT(oop_result1, metadata_result, entry, 2);
 143 }
 144 
 145 
 146 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
 147   // if there is any conflict use the stack
 148   if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
 149       arg2 == c_rarg1 || arg1 == c_rarg3 ||
 150       arg3 == c_rarg1 || arg1 == c_rarg2) {
 151     stp(arg3, arg2, Address(pre(sp, 2 * wordSize)));
 152     stp(arg1, zr, Address(pre(sp, -2 * wordSize)));
 153     ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));
 154     ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));
 155   } else {
 156     mov(c_rarg1, arg1);
 157     mov(c_rarg2, arg2);
 158     mov(c_rarg3, arg3);
 159   }
 160   return call_RT(oop_result1, metadata_result, entry, 3);
 161 }
 162 
 163 // Implementation of StubFrame
 164 
 165 class StubFrame: public StackObj {
 166  private:
 167   StubAssembler* _sasm;
 168 
 169  public:
 170   StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
 171   void load_argument(int offset_in_words, Register reg);
 172 
 173   ~StubFrame();
 174 };;
 175 
 176 
 177 #define __ _sasm->
 178 
 179 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {
 180   _sasm = sasm;
 181   __ set_info(name, must_gc_arguments);
 182   __ enter();
 183 }
 184 
 185 // load parameters that were stored with LIR_Assembler::store_parameter
 186 // Note: offsets for store_parameter and load_argument must match
 187 void StubFrame::load_argument(int offset_in_words, Register reg) {
 188   // rbp, + 0: link
 189   //     + 1: return address
 190   //     + 2: argument with offset 0
 191   //     + 3: argument with offset 1
 192   //     + 4: ...
 193 
 194   __ ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord));
 195 }
 196 
 197 
 198 StubFrame::~StubFrame() {
 199   __ leave();
 200   __ ret(lr);
 201 }
 202 
 203 #undef __
 204 
 205 
 206 // Implementation of Runtime1
 207 
 208 #define __ sasm->
 209 
 210 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
 211 
 212 // Stack layout for saving/restoring  all the registers needed during a runtime
 213 // call (this includes deoptimization)
 214 // Note: note that users of this frame may well have arguments to some runtime
 215 // while these values are on the stack. These positions neglect those arguments
 216 // but the code in save_live_registers will take the argument count into
 217 // account.
 218 //
 219 
 220 enum reg_save_layout {
 221   reg_save_frame_size = 32 /* float */ + 32 /* integer */
 222 };
 223 
 224 // Save off registers which might be killed by calls into the runtime.
 225 // Tries to smart of about FP registers.  In particular we separate
 226 // saving and describing the FPU registers for deoptimization since we
 227 // have to save the FPU registers twice if we describe them.  The
 228 // deopt blob is the only thing which needs to describe FPU registers.
 229 // In all other cases it should be sufficient to simply save their
 230 // current value.
 231 
 232 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
 233 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
 234 static int reg_save_size_in_words;
 235 static int frame_size_in_bytes = -1;
 236 
 237 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
 238   int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
 239   sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
 240   int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
 241   OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
 242 
 243   for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {
 244     Register r = as_Register(i);
 245     if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) {
 246       int sp_offset = cpu_reg_save_offsets[i];
 247       oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 248                                 r->as_VMReg());
 249     }
 250   }
 251 
 252   if (save_fpu_registers) {
 253     for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
 254       FloatRegister r = as_FloatRegister(i);
 255       {
 256         int sp_offset = fpu_reg_save_offsets[i];
 257         oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 258                                   r->as_VMReg());
 259       }
 260     }
 261   }
 262   return oop_map;
 263 }
 264 
 265 static OopMap* save_live_registers(StubAssembler* sasm,
 266                                    bool save_fpu_registers = true) {
 267   __ block_comment("save_live_registers");
 268 
 269   __ push(RegSet::range(r0, r29), sp);         // integer registers except lr & sp
 270 
 271   if (save_fpu_registers) {
 272     for (int i = 30; i >= 0; i -= 2)
 273       __ stpd(as_FloatRegister(i), as_FloatRegister(i+1),
 274               Address(__ pre(sp, -2 * wordSize)));
 275   } else {
 276     __ add(sp, sp, -32 * wordSize);
 277   }
 278 
 279   return generate_oop_map(sasm, save_fpu_registers);
 280 }
 281 
 282 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
 283   if (restore_fpu_registers) {
 284     for (int i = 0; i < 32; i += 2)
 285       __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
 286               Address(__ post(sp, 2 * wordSize)));
 287   } else {
 288     __ add(sp, sp, 32 * wordSize);
 289   }
 290 
 291   __ pop(RegSet::range(r0, r29), sp);
 292 }
 293 
 294 static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true)  {
 295 
 296   if (restore_fpu_registers) {
 297     for (int i = 0; i < 32; i += 2)
 298       __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
 299               Address(__ post(sp, 2 * wordSize)));
 300   } else {
 301     __ add(sp, sp, 32 * wordSize);
 302   }
 303 
 304   __ ldp(zr, r1, Address(__ post(sp, 16)));
 305   __ pop(RegSet::range(r2, r29), sp);
 306 }
 307 
 308 
 309 
 310 void Runtime1::initialize_pd() {
 311   int i;
 312   int sp_offset = 0;
 313 
 314   // all float registers are saved explicitly
 315   assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");
 316   for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
 317     fpu_reg_save_offsets[i] = sp_offset;
 318     sp_offset += 2;   // SP offsets are in halfwords
 319   }
 320 
 321   for (i = 0; i < FrameMap::nof_cpu_regs; i++) {
 322     Register r = as_Register(i);
 323     cpu_reg_save_offsets[i] = sp_offset;
 324     sp_offset += 2;   // SP offsets are in halfwords
 325   }
 326 }
 327 
 328 
 329 // target: the entry point of the method that creates and posts the exception oop
 330 // has_argument: true if the exception needs an argument (passed in rscratch1)
 331 
 332 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
 333   // make a frame and preserve the caller's caller-save registers
 334   OopMap* oop_map = save_live_registers(sasm);
 335   int call_offset;
 336   if (!has_argument) {
 337     call_offset = __ call_RT(noreg, noreg, target);
 338   } else {
 339     call_offset = __ call_RT(noreg, noreg, target, rscratch1);
 340   }
 341   OopMapSet* oop_maps = new OopMapSet();
 342   oop_maps->add_gc_map(call_offset, oop_map);
 343 
 344   __ should_not_reach_here();
 345   return oop_maps;
 346 }
 347 
 348 
 349 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
 350   __ block_comment("generate_handle_exception");
 351 
 352   // incoming parameters
 353   const Register exception_oop = r0;
 354   const Register exception_pc  = r3;
 355   // other registers used in this stub
 356 
 357   // Save registers, if required.
 358   OopMapSet* oop_maps = new OopMapSet();
 359   OopMap* oop_map = NULL;
 360   switch (id) {
 361   case forward_exception_id:
 362     // We're handling an exception in the context of a compiled frame.
 363     // The registers have been saved in the standard places.  Perform
 364     // an exception lookup in the caller and dispatch to the handler
 365     // if found.  Otherwise unwind and dispatch to the callers
 366     // exception handler.
 367     oop_map = generate_oop_map(sasm, 1 /*thread*/);
 368 
 369     // load and clear pending exception oop into r0
 370     __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));
 371     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 372 
 373     // load issuing PC (the return address for this stub) into r3
 374     __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
 375 
 376     // make sure that the vm_results are cleared (may be unnecessary)
 377     __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
 378     __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
 379     break;
 380   case handle_exception_nofpu_id:
 381   case handle_exception_id:
 382     // At this point all registers MAY be live.
 383     oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
 384     break;
 385   case handle_exception_from_callee_id: {
 386     // At this point all registers except exception oop (r0) and
 387     // exception pc (lr) are dead.
 388     const int frame_size = 2 /*fp, return address*/;
 389     oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
 390     sasm->set_frame_size(frame_size);
 391     break;
 392   }
 393   default:
 394     __ should_not_reach_here();
 395     break;
 396   }
 397 
 398   // verify that only r0 and r3 are valid at this time
 399   __ invalidate_registers(false, true, true, false, true, true);
 400   // verify that r0 contains a valid exception
 401   __ verify_not_null_oop(exception_oop);
 402 
 403 #ifdef ASSERT
 404   // check that fields in JavaThread for exception oop and issuing pc are
 405   // empty before writing to them
 406   Label oop_empty;
 407   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 408   __ cbz(rscratch1, oop_empty);
 409   __ stop("exception oop already set");
 410   __ bind(oop_empty);
 411 
 412   Label pc_empty;
 413   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 414   __ cbz(rscratch1, pc_empty);
 415   __ stop("exception pc already set");
 416   __ bind(pc_empty);
 417 #endif
 418 
 419   // save exception oop and issuing pc into JavaThread
 420   // (exception handler will load it from here)
 421   __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));
 422   __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
 423 
 424   // patch throwing pc into return address (has bci & oop map)
 425   __ str(exception_pc, Address(rfp, 1*BytesPerWord));
 426 
 427   // compute the exception handler.
 428   // the exception oop and the throwing pc are read from the fields in JavaThread
 429   int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
 430   oop_maps->add_gc_map(call_offset, oop_map);
 431 
 432   // r0: handler address
 433   //      will be the deopt blob if nmethod was deoptimized while we looked up
 434   //      handler regardless of whether handler existed in the nmethod.
 435 
 436   // only r0 is valid at this time, all other registers have been destroyed by the runtime call
 437   __ invalidate_registers(false, true, true, true, true, true);
 438 
 439   // patch the return address, this stub will directly return to the exception handler
 440   __ str(r0, Address(rfp, 1*BytesPerWord));
 441 
 442   switch (id) {
 443   case forward_exception_id:
 444   case handle_exception_nofpu_id:
 445   case handle_exception_id:
 446     // Restore the registers that were saved at the beginning.
 447     restore_live_registers(sasm, id != handle_exception_nofpu_id);
 448     break;
 449   case handle_exception_from_callee_id:
 450     // Pop the return address.
 451     __ leave();
 452     __ ret(lr);  // jump to exception handler
 453     break;
 454   default:  ShouldNotReachHere();
 455   }
 456 
 457   return oop_maps;
 458 }
 459 
 460 
 461 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
 462   // incoming parameters
 463   const Register exception_oop = r0;
 464   // callee-saved copy of exception_oop during runtime call
 465   const Register exception_oop_callee_saved = r19;
 466   // other registers used in this stub
 467   const Register exception_pc = r3;
 468   const Register handler_addr = r1;
 469 
 470   // verify that only r0, is valid at this time
 471   __ invalidate_registers(false, true, true, true, true, true);
 472 
 473 #ifdef ASSERT
 474   // check that fields in JavaThread for exception oop and issuing pc are empty
 475   Label oop_empty;
 476   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 477   __ cbz(rscratch1, oop_empty);
 478   __ stop("exception oop must be empty");
 479   __ bind(oop_empty);
 480 
 481   Label pc_empty;
 482   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 483   __ cbz(rscratch1, pc_empty);
 484   __ stop("exception pc must be empty");
 485   __ bind(pc_empty);
 486 #endif
 487 
 488   // Save our return address because
 489   // exception_handler_for_return_address will destroy it.  We also
 490   // save exception_oop
 491   __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
 492 
 493   // search the exception handler address of the caller (using the return address)
 494   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr);
 495   // r0: exception handler address of the caller
 496 
 497   // Only R0 is valid at this time; all other registers have been
 498   // destroyed by the call.
 499   __ invalidate_registers(false, true, true, true, false, true);
 500 
 501   // move result of call into correct register
 502   __ mov(handler_addr, r0);
 503 
 504   // get throwing pc (= return address).
 505   // lr has been destroyed by the call
 506   __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
 507   __ mov(r3, lr);
 508 
 509   __ verify_not_null_oop(exception_oop);
 510 
 511   // continue at exception handler (return address removed)
 512   // note: do *not* remove arguments when unwinding the
 513   //       activation since the caller assumes having
 514   //       all arguments on the stack when entering the
 515   //       runtime to determine the exception handler
 516   //       (GC happens at call site with arguments!)
 517   // r0: exception oop
 518   // r3: throwing pc
 519   // r1: exception handler
 520   __ br(handler_addr);
 521 }
 522 
 523 
 524 
 525 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
 526   // use the maximum number of runtime-arguments here because it is difficult to
 527   // distinguish each RT-Call.
 528   // Note: This number affects also the RT-Call in generate_handle_exception because
 529   //       the oop-map is shared for all calls.
 530   DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
 531   assert(deopt_blob != NULL, "deoptimization blob must have been created");
 532 
 533   OopMap* oop_map = save_live_registers(sasm);
 534 
 535   __ mov(c_rarg0, rthread);
 536   Label retaddr;
 537   __ set_last_Java_frame(sp, rfp, retaddr, rscratch1);
 538   // do the call
 539   __ lea(rscratch1, RuntimeAddress(target));
 540   __ blrt(rscratch1, 1, 0, 1);
 541   __ bind(retaddr);
 542   OopMapSet* oop_maps = new OopMapSet();
 543   oop_maps->add_gc_map(__ offset(), oop_map);
 544   // verify callee-saved register
 545 #ifdef ASSERT
 546   { Label L;
 547     __ get_thread(rscratch1);
 548     __ cmp(rthread, rscratch1);
 549     __ br(Assembler::EQ, L);
 550     __ stop("StubAssembler::call_RT: rthread not callee saved?");
 551     __ bind(L);
 552   }
 553 #endif
 554   __ reset_last_Java_frame(true);
 555   __ maybe_isb();
 556 
 557   // check for pending exceptions
 558   { Label L;
 559     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 560     __ cbz(rscratch1, L);
 561     // exception pending => remove activation and forward to exception handler
 562 
 563     { Label L1;
 564       __ cbnz(r0, L1);                                  // have we deoptimized?
 565       __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 566       __ bind(L1);
 567     }
 568 
 569     // the deopt blob expects exceptions in the special fields of
 570     // JavaThread, so copy and clear pending exception.
 571 
 572     // load and clear pending exception
 573     __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 574     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 575 
 576     // check that there is really a valid exception
 577     __ verify_not_null_oop(r0);
 578 
 579     // load throwing pc: this is the return address of the stub
 580     __ mov(r3, lr);
 581 
 582 #ifdef ASSERT
 583     // check that fields in JavaThread for exception oop and issuing pc are empty
 584     Label oop_empty;
 585     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 586     __ cbz(rscratch1, oop_empty);
 587     __ stop("exception oop must be empty");
 588     __ bind(oop_empty);
 589 
 590     Label pc_empty;
 591     __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 592     __ cbz(rscratch1, pc_empty);
 593     __ stop("exception pc must be empty");
 594     __ bind(pc_empty);
 595 #endif
 596 
 597     // store exception oop and throwing pc to JavaThread
 598     __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
 599     __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
 600 
 601     restore_live_registers(sasm);
 602 
 603     __ leave();
 604 
 605     // Forward the exception directly to deopt blob. We can blow no
 606     // registers and must leave throwing pc on the stack.  A patch may
 607     // have values live in registers so the entry point with the
 608     // exception in tls.
 609     __ far_jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls()));
 610 
 611     __ bind(L);
 612   }
 613 
 614 
 615   // Runtime will return true if the nmethod has been deoptimized during
 616   // the patching process. In that case we must do a deopt reexecute instead.
 617 
 618   Label reexecuteEntry, cont;
 619 
 620   __ cbz(r0, cont);                                 // have we deoptimized?
 621 
 622   // Will reexecute. Proper return address is already on the stack we just restore
 623   // registers, pop all of our frame but the return address and jump to the deopt blob
 624   restore_live_registers(sasm);
 625   __ leave();
 626   __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
 627 
 628   __ bind(cont);
 629   restore_live_registers(sasm);
 630   __ leave();
 631   __ ret(lr);
 632 
 633   return oop_maps;
 634 }
 635 
 636 
 637 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
 638 
 639   const Register exception_oop = r0;
 640   const Register exception_pc  = r3;
 641 
 642   // for better readability
 643   const bool must_gc_arguments = true;
 644   const bool dont_gc_arguments = false;
 645 
 646   // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
 647   bool save_fpu_registers = true;
 648 
 649   // stub code & info for the different stubs
 650   OopMapSet* oop_maps = NULL;
 651   OopMap* oop_map = NULL;
 652   switch (id) {
 653     {
 654     case forward_exception_id:
 655       {
 656         oop_maps = generate_handle_exception(id, sasm);
 657         __ leave();
 658         __ ret(lr);
 659       }
 660       break;
 661 
 662     case throw_div0_exception_id:
 663       { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments);
 664         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
 665       }
 666       break;
 667 
 668     case throw_null_pointer_exception_id:
 669       { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments);
 670         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
 671       }
 672       break;
 673 
 674     case new_instance_id:
 675     case fast_new_instance_id:
 676     case fast_new_instance_init_check_id:
 677       {
 678         Register klass = r3; // Incoming
 679         Register obj   = r0; // Result
 680 
 681         if (id == new_instance_id) {
 682           __ set_info("new_instance", dont_gc_arguments);
 683         } else if (id == fast_new_instance_id) {
 684           __ set_info("fast new_instance", dont_gc_arguments);
 685         } else {
 686           assert(id == fast_new_instance_init_check_id, "bad StubID");
 687           __ set_info("fast new_instance init check", dont_gc_arguments);
 688         }
 689 
 690         if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
 691             UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
 692           Label slow_path;
 693           Register obj_size = r2;
 694           Register t1       = r19;
 695           Register t2       = r4;
 696           assert_different_registers(klass, obj, obj_size, t1, t2);
 697 
 698           __ stp(r19, zr, Address(__ pre(sp, -2 * wordSize)));
 699 
 700           if (id == fast_new_instance_init_check_id) {
 701             // make sure the klass is initialized
 702             __ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset()));
 703             __ cmpw(rscratch1, InstanceKlass::fully_initialized);
 704             __ br(Assembler::NE, slow_path);
 705           }
 706 
 707 #ifdef ASSERT
 708           // assert object can be fast path allocated
 709           {
 710             Label ok, not_ok;
 711             __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 712             __ cmp(obj_size, 0u);
 713             __ br(Assembler::LE, not_ok);  // make sure it's an instance (LH > 0)
 714             __ tstw(obj_size, Klass::_lh_instance_slow_path_bit);
 715             __ br(Assembler::EQ, ok);
 716             __ bind(not_ok);
 717             __ stop("assert(can be fast path allocated)");
 718             __ should_not_reach_here();
 719             __ bind(ok);
 720           }
 721 #endif // ASSERT
 722 
 723           // get the instance size (size is postive so movl is fine for 64bit)
 724           __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 725 
 726           __ eden_allocate(obj, obj_size, 0, t1, slow_path);
 727           __ incr_allocated_bytes(rthread, obj_size, 0, rscratch1);
 728 
 729           __ initialize_object(obj, klass, obj_size, 0, t1, t2, /* is_tlab_allocated */ false);
 730           __ verify_oop(obj);
 731           __ ldp(r19, zr, Address(__ post(sp, 2 * wordSize)));
 732           __ ret(lr);
 733 
 734           __ bind(slow_path);
 735           __ ldp(r19, zr, Address(__ post(sp, 2 * wordSize)));
 736         }
 737 
 738         __ enter();
 739         OopMap* map = save_live_registers(sasm);
 740         int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
 741         oop_maps = new OopMapSet();
 742         oop_maps->add_gc_map(call_offset, map);
 743         restore_live_registers_except_r0(sasm);
 744         __ verify_oop(obj);
 745         __ leave();
 746         __ ret(lr);
 747 
 748         // r0,: new instance
 749       }
 750 
 751       break;
 752 
 753     case counter_overflow_id:
 754       {
 755         Register bci = r0, method = r1;
 756         __ enter();
 757         OopMap* map = save_live_registers(sasm);
 758         // Retrieve bci
 759         __ ldrw(bci, Address(rfp, 2*BytesPerWord));
 760         // And a pointer to the Method*
 761         __ ldr(method, Address(rfp, 3*BytesPerWord));
 762         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
 763         oop_maps = new OopMapSet();
 764         oop_maps->add_gc_map(call_offset, map);
 765         restore_live_registers(sasm);
 766         __ leave();
 767         __ ret(lr);
 768       }
 769       break;
 770 
 771     case new_type_array_id:
 772     case new_object_array_id:
 773       {
 774         Register length   = r19; // Incoming
 775         Register klass    = r3; // Incoming
 776         Register obj      = r0; // Result
 777 
 778         if (id == new_type_array_id) {
 779           __ set_info("new_type_array", dont_gc_arguments);
 780         } else {
 781           __ set_info("new_object_array", dont_gc_arguments);
 782         }
 783 
 784 #ifdef ASSERT
 785         // assert object type is really an array of the proper kind
 786         {
 787           Label ok;
 788           Register t0 = obj;
 789           __ ldrw(t0, Address(klass, Klass::layout_helper_offset()));
 790           __ asrw(t0, t0, Klass::_lh_array_tag_shift);
 791           int tag = ((id == new_type_array_id)
 792                      ? Klass::_lh_array_tag_type_value
 793                      : Klass::_lh_array_tag_obj_value);
 794           __ mov(rscratch1, tag);
 795           __ cmpw(t0, rscratch1);
 796           __ br(Assembler::EQ, ok);
 797           __ stop("assert(is an array klass)");
 798           __ should_not_reach_here();
 799           __ bind(ok);
 800         }
 801 #endif // ASSERT
 802 
 803         if (UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
 804           Register arr_size = r4;
 805           Register t1       = r2;
 806           Register t2       = r5;
 807           Label slow_path;
 808           assert_different_registers(length, klass, obj, arr_size, t1, t2);
 809 
 810           // check that array length is small enough for fast path.
 811           __ mov(rscratch1, C1_MacroAssembler::max_array_allocation_length);
 812           __ cmpw(length, rscratch1);
 813           __ br(Assembler::HI, slow_path);
 814 
 815           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
 816           // since size is positive ldrw does right thing on 64bit
 817           __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
 818           // since size is positive movw does right thing on 64bit
 819           __ movw(arr_size, length);
 820           __ lslvw(arr_size, length, t1);
 821           __ ubfx(t1, t1, Klass::_lh_header_size_shift,
 822                   exact_log2(Klass::_lh_header_size_mask + 1));
 823           __ add(arr_size, arr_size, t1);
 824           __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
 825           __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 826 
 827           __ eden_allocate(obj, arr_size, 0, t1, slow_path);  // preserves arr_size
 828           __ incr_allocated_bytes(rthread, arr_size, 0, rscratch1);
 829 
 830           __ initialize_header(obj, klass, length, t1, t2);
 831           __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
 832           assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
 833           assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
 834           __ andr(t1, t1, Klass::_lh_header_size_mask);
 835           __ sub(arr_size, arr_size, t1);  // body length
 836           __ add(t1, t1, obj);       // body start
 837           __ initialize_body(t1, arr_size, 0, t2);
 838           __ verify_oop(obj);
 839 
 840           __ ret(lr);
 841 
 842           __ bind(slow_path);
 843         }
 844 
 845         __ enter();
 846         OopMap* map = save_live_registers(sasm);
 847         int call_offset;
 848         if (id == new_type_array_id) {
 849           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
 850         } else {
 851           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
 852         }
 853 
 854         oop_maps = new OopMapSet();
 855         oop_maps->add_gc_map(call_offset, map);
 856         restore_live_registers_except_r0(sasm);
 857 
 858         __ verify_oop(obj);
 859         __ leave();
 860         __ ret(lr);
 861 
 862         // r0: new array
 863       }
 864       break;
 865 
 866     case new_multi_array_id:
 867       { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
 868         // r0,: klass
 869         // r19,: rank
 870         // r2: address of 1st dimension
 871         OopMap* map = save_live_registers(sasm);
 872         __ mov(c_rarg1, r0);
 873         __ mov(c_rarg3, r2);
 874         __ mov(c_rarg2, r19);
 875         int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3);
 876 
 877         oop_maps = new OopMapSet();
 878         oop_maps->add_gc_map(call_offset, map);
 879         restore_live_registers_except_r0(sasm);
 880 
 881         // r0,: new multi array
 882         __ verify_oop(r0);
 883       }
 884       break;
 885 
 886     case register_finalizer_id:
 887       {
 888         __ set_info("register_finalizer", dont_gc_arguments);
 889 
 890         // This is called via call_runtime so the arguments
 891         // will be place in C abi locations
 892 
 893         __ verify_oop(c_rarg0);
 894 
 895         // load the klass and check the has finalizer flag
 896         Label register_finalizer;
 897         Register t = r5;
 898         __ load_klass(t, r0);
 899         __ ldrw(t, Address(t, Klass::access_flags_offset()));
 900         __ tbnz(t, exact_log2(JVM_ACC_HAS_FINALIZER), register_finalizer);
 901         __ ret(lr);
 902 
 903         __ bind(register_finalizer);
 904         __ enter();
 905         OopMap* oop_map = save_live_registers(sasm);
 906         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0);
 907         oop_maps = new OopMapSet();
 908         oop_maps->add_gc_map(call_offset, oop_map);
 909 
 910         // Now restore all the live registers
 911         restore_live_registers(sasm);
 912 
 913         __ leave();
 914         __ ret(lr);
 915       }
 916       break;
 917 
 918     case throw_class_cast_exception_id:
 919       { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments);
 920         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
 921       }
 922       break;
 923 
 924     case throw_incompatible_class_change_error_id:
 925       { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments);
 926         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
 927       }
 928       break;
 929 
 930     case slow_subtype_check_id:
 931       {
 932         // Typical calling sequence:
 933         // __ push(klass_RInfo);  // object klass or other subclass
 934         // __ push(sup_k_RInfo);  // array element klass or other superclass
 935         // __ bl(slow_subtype_check);
 936         // Note that the subclass is pushed first, and is therefore deepest.
 937         enum layout {
 938           r0_off, r0_off_hi,
 939           r2_off, r2_off_hi,
 940           r4_off, r4_off_hi,
 941           r5_off, r5_off_hi,
 942           sup_k_off, sup_k_off_hi,
 943           klass_off, klass_off_hi,
 944           framesize,
 945           result_off = sup_k_off
 946         };
 947 
 948         __ set_info("slow_subtype_check", dont_gc_arguments);
 949         __ push(RegSet::of(r0, r2, r4, r5), sp);
 950 
 951         // This is called by pushing args and not with C abi
 952         // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
 953         // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
 954 
 955         __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
 956 
 957         Label miss;
 958         __ check_klass_subtype_slow_path(r4, r0, r2, r5, NULL, &miss);
 959 
 960         // fallthrough on success:
 961         __ mov(rscratch1, 1);
 962         __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
 963         __ pop(RegSet::of(r0, r2, r4, r5), sp);
 964         __ ret(lr);
 965 
 966         __ bind(miss);
 967         __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
 968         __ pop(RegSet::of(r0, r2, r4, r5), sp);
 969         __ ret(lr);
 970       }
 971       break;
 972 
 973     case monitorenter_nofpu_id:
 974       save_fpu_registers = false;
 975       // fall through
 976     case monitorenter_id:
 977       {
 978         StubFrame f(sasm, "monitorenter", dont_gc_arguments);
 979         OopMap* map = save_live_registers(sasm, save_fpu_registers);
 980 
 981         // Called with store_parameter and not C abi
 982 
 983         f.load_argument(1, r0); // r0,: object
 984         f.load_argument(0, r1); // r1,: lock address
 985 
 986         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1);
 987 
 988         oop_maps = new OopMapSet();
 989         oop_maps->add_gc_map(call_offset, map);
 990         restore_live_registers(sasm, save_fpu_registers);
 991       }
 992       break;
 993 
 994     case monitorexit_nofpu_id:
 995       save_fpu_registers = false;
 996       // fall through
 997     case monitorexit_id:
 998       {
 999         StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1000         OopMap* map = save_live_registers(sasm, save_fpu_registers);
1001 
1002         // Called with store_parameter and not C abi
1003 
1004         f.load_argument(0, r0); // r0,: lock address
1005 
1006         // note: really a leaf routine but must setup last java sp
1007         //       => use call_RT for now (speed can be improved by
1008         //       doing last java sp setup manually)
1009         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0);
1010 
1011         oop_maps = new OopMapSet();
1012         oop_maps->add_gc_map(call_offset, map);
1013         restore_live_registers(sasm, save_fpu_registers);
1014       }
1015       break;
1016 
1017     case deoptimize_id:
1018       {
1019         StubFrame f(sasm, "deoptimize", dont_gc_arguments);
1020         OopMap* oop_map = save_live_registers(sasm);
1021         f.load_argument(0, c_rarg1);
1022         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1);
1023 
1024         oop_maps = new OopMapSet();
1025         oop_maps->add_gc_map(call_offset, oop_map);
1026         restore_live_registers(sasm);
1027         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1028         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1029         __ leave();
1030         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1031       }
1032       break;
1033 
1034     case throw_range_check_failed_id:
1035       { StubFrame f(sasm, "range_check_failed", dont_gc_arguments);
1036         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1037       }
1038       break;
1039 
1040     case unwind_exception_id:
1041       { __ set_info("unwind_exception", dont_gc_arguments);
1042         // note: no stubframe since we are about to leave the current
1043         //       activation and we are calling a leaf VM function only.
1044         generate_unwind_exception(sasm);
1045       }
1046       break;
1047 
1048     case access_field_patching_id:
1049       { StubFrame f(sasm, "access_field_patching", dont_gc_arguments);
1050         // we should set up register map
1051         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1052       }
1053       break;
1054 
1055     case load_klass_patching_id:
1056       { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments);
1057         // we should set up register map
1058         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1059       }
1060       break;
1061 
1062     case load_mirror_patching_id:
1063       { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments);
1064         // we should set up register map
1065         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1066       }
1067       break;
1068 
1069     case load_appendix_patching_id:
1070       { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments);
1071         // we should set up register map
1072         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1073       }
1074       break;
1075 
1076     case handle_exception_nofpu_id:
1077     case handle_exception_id:
1078       { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1079         oop_maps = generate_handle_exception(id, sasm);
1080       }
1081       break;
1082 
1083     case handle_exception_from_callee_id:
1084       { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1085         oop_maps = generate_handle_exception(id, sasm);
1086       }
1087       break;
1088 
1089     case throw_index_exception_id:
1090       { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments);
1091         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1092       }
1093       break;
1094 
1095     case throw_array_store_exception_id:
1096       { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments);
1097         // tos + 0: link
1098         //     + 1: return address
1099         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1100       }
1101       break;
1102 
1103 #if INCLUDE_ALL_GCS
1104 
1105     case g1_pre_barrier_slow_id:
1106       {
1107         StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments);
1108         // arg0 : previous value of memory
1109 
1110         BarrierSet* bs = BarrierSet::barrier_set();
1111         if (bs->kind() != BarrierSet::G1BarrierSet) {
1112           __ mov(r0, (int)id);
1113           __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1114           __ should_not_reach_here();
1115           break;
1116         }
1117 
1118         const Register pre_val = r0;
1119         const Register thread = rthread;
1120         const Register tmp = rscratch1;
1121 
1122         Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
1123         Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
1124         Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
1125 
1126         Label done;
1127         Label runtime;
1128 
1129         // Is marking still active?
1130         if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
1131           __ ldrw(tmp, in_progress);
1132         } else {
1133           assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
1134           __ ldrb(tmp, in_progress);
1135         }
1136         __ cbzw(tmp, done);
1137 
1138         // Can we store original value in the thread's buffer?
1139         __ ldr(tmp, queue_index);
1140         __ cbz(tmp, runtime);
1141 
1142         __ sub(tmp, tmp, wordSize);
1143         __ str(tmp, queue_index);
1144         __ ldr(rscratch2, buffer);
1145         __ add(tmp, tmp, rscratch2);
1146         f.load_argument(0, rscratch2);
1147         __ str(rscratch2, Address(tmp, 0));
1148         __ b(done);
1149 
1150         __ bind(runtime);
1151         __ push_call_clobbered_registers();
1152         f.load_argument(0, pre_val);
1153         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
1154         __ pop_call_clobbered_registers();
1155         __ bind(done);
1156       }
1157       break;
1158     case g1_post_barrier_slow_id:
1159       {
1160         StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments);
1161 
1162         BarrierSet* bs = BarrierSet::barrier_set();
1163         if (bs->kind() != BarrierSet::G1BarrierSet) {
1164           __ mov(r0, (int)id);
1165           __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1166           __ should_not_reach_here();
1167           break;
1168         }
1169 
1170         // arg0: store_address
1171         Address store_addr(rfp, 2*BytesPerWord);
1172 
1173         Label done;
1174         Label runtime;
1175 
1176         // At this point we know new_value is non-NULL and the new_value crosses regions.
1177         // Must check to see if card is already dirty
1178 
1179         const Register thread = rthread;
1180 
1181         Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
1182         Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
1183 
1184         const Register card_offset = rscratch2;
1185         // LR is free here, so we can use it to hold the byte_map_base.
1186         const Register byte_map_base = lr;
1187 
1188         assert_different_registers(card_offset, byte_map_base, rscratch1);
1189 
1190         f.load_argument(0, card_offset);
1191         __ lsr(card_offset, card_offset, CardTable::card_shift);
1192         __ load_byte_map_base(byte_map_base);
1193         __ ldrb(rscratch1, Address(byte_map_base, card_offset));
1194         __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val());
1195         __ br(Assembler::EQ, done);
1196 
1197         assert((int)CardTable::dirty_card_val() == 0, "must be 0");
1198 
1199         __ membar(Assembler::StoreLoad);
1200         __ ldrb(rscratch1, Address(byte_map_base, card_offset));
1201         __ cbzw(rscratch1, done);
1202 
1203         // storing region crossing non-NULL, card is clean.
1204         // dirty card and log.
1205         __ strb(zr, Address(byte_map_base, card_offset));
1206 
1207         // Convert card offset into an address in card_addr
1208         Register card_addr = card_offset;
1209         __ add(card_addr, byte_map_base, card_addr);
1210 
1211         __ ldr(rscratch1, queue_index);
1212         __ cbz(rscratch1, runtime);
1213         __ sub(rscratch1, rscratch1, wordSize);
1214         __ str(rscratch1, queue_index);
1215 
1216         // Reuse LR to hold buffer_addr
1217         const Register buffer_addr = lr;
1218 
1219         __ ldr(buffer_addr, buffer);
1220         __ str(card_addr, Address(buffer_addr, rscratch1));
1221         __ b(done);
1222 
1223         __ bind(runtime);
1224         __ push_call_clobbered_registers();
1225         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
1226         __ pop_call_clobbered_registers();
1227         __ bind(done);
1228 
1229       }
1230       break;
1231 #endif
1232 
1233     case predicate_failed_trap_id:
1234       {
1235         StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1236 
1237         OopMap* map = save_live_registers(sasm);
1238 
1239         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1240         oop_maps = new OopMapSet();
1241         oop_maps->add_gc_map(call_offset, map);
1242         restore_live_registers(sasm);
1243         __ leave();
1244         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1245         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1246 
1247         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1248       }
1249       break;
1250 
1251 
1252     default:
1253       { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
1254         __ mov(r0, (int)id);
1255         __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1256         __ should_not_reach_here();
1257       }
1258       break;
1259     }
1260   }
1261   return oop_maps;
1262 }
1263 
1264 #undef __
1265 
1266 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }
--- EOF ---