rev 51052 : [mq]: eden

   1 /*
   2  * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/assembler.hpp"
  28 #include "c1/c1_CodeStubs.hpp"
  29 #include "c1/c1_Defs.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "compiler/disassembler.hpp"
  33 #include "gc/shared/cardTable.hpp"
  34 #include "gc/shared/cardTableBarrierSet.hpp"
  35 #include "interpreter/interpreter.hpp"
  36 #include "nativeInst_aarch64.hpp"
  37 #include "oops/compiledICHolder.hpp"
  38 #include "oops/oop.inline.hpp"
  39 #include "prims/jvmtiExport.hpp"
  40 #include "register_aarch64.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 #include "runtime/signature.hpp"
  43 #include "runtime/vframe.hpp"
  44 #include "runtime/vframeArray.hpp"
  45 #include "vmreg_aarch64.inline.hpp"
  46 
  47 
  48 // Implementation of StubAssembler
  49 
  50 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
  51   // setup registers
  52   assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
  53   assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");
  54   assert(args_size >= 0, "illegal args_size");
  55   bool align_stack = false;
  56 
  57   mov(c_rarg0, rthread);
  58   set_num_rt_args(0); // Nothing on stack
  59 
  60   Label retaddr;
  61   set_last_Java_frame(sp, rfp, retaddr, rscratch1);
  62 
  63   // do the call
  64   lea(rscratch1, RuntimeAddress(entry));
  65   blrt(rscratch1, args_size + 1, 8, 1);
  66   bind(retaddr);
  67   int call_offset = offset();
  68   // verify callee-saved register
  69 #ifdef ASSERT
  70   push(r0, sp);
  71   { Label L;
  72     get_thread(r0);
  73     cmp(rthread, r0);
  74     br(Assembler::EQ, L);
  75     stop("StubAssembler::call_RT: rthread not callee saved?");
  76     bind(L);
  77   }
  78   pop(r0, sp);
  79 #endif
  80   reset_last_Java_frame(true);
  81   maybe_isb();
  82 
  83   // check for pending exceptions
  84   { Label L;
  85     // check for pending exceptions (java_thread is set upon return)
  86     ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
  87     cbz(rscratch1, L);
  88     // exception pending => remove activation and forward to exception handler
  89     // make sure that the vm_results are cleared
  90     if (oop_result1->is_valid()) {
  91       str(zr, Address(rthread, JavaThread::vm_result_offset()));
  92     }
  93     if (metadata_result->is_valid()) {
  94       str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
  95     }
  96     if (frame_size() == no_frame_size) {
  97       leave();
  98       far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
  99     } else if (_stub_id == Runtime1::forward_exception_id) {
 100       should_not_reach_here();
 101     } else {
 102       far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 103     }
 104     bind(L);
 105   }
 106   // get oop results if there are any and reset the values in the thread
 107   if (oop_result1->is_valid()) {
 108     get_vm_result(oop_result1, rthread);
 109   }
 110   if (metadata_result->is_valid()) {
 111     get_vm_result_2(metadata_result, rthread);
 112   }
 113   return call_offset;
 114 }
 115 
 116 
 117 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
 118   mov(c_rarg1, arg1);
 119   return call_RT(oop_result1, metadata_result, entry, 1);
 120 }
 121 
 122 
 123 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
 124   if (c_rarg1 == arg2) {
 125     if (c_rarg2 == arg1) {
 126       mov(rscratch1, arg1);
 127       mov(arg1, arg2);
 128       mov(arg2, rscratch1);
 129     } else {
 130       mov(c_rarg2, arg2);
 131       mov(c_rarg1, arg1);
 132     }
 133   } else {
 134     mov(c_rarg1, arg1);
 135     mov(c_rarg2, arg2);
 136   }
 137   return call_RT(oop_result1, metadata_result, entry, 2);
 138 }
 139 
 140 
 141 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
 142   // if there is any conflict use the stack
 143   if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
 144       arg2 == c_rarg1 || arg1 == c_rarg3 ||
 145       arg3 == c_rarg1 || arg1 == c_rarg2) {
 146     stp(arg3, arg2, Address(pre(sp, 2 * wordSize)));
 147     stp(arg1, zr, Address(pre(sp, -2 * wordSize)));
 148     ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));
 149     ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));
 150   } else {
 151     mov(c_rarg1, arg1);
 152     mov(c_rarg2, arg2);
 153     mov(c_rarg3, arg3);
 154   }
 155   return call_RT(oop_result1, metadata_result, entry, 3);
 156 }
 157 
 158 // Implementation of StubFrame
 159 
 160 class StubFrame: public StackObj {
 161  private:
 162   StubAssembler* _sasm;
 163 
 164  public:
 165   StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
 166   void load_argument(int offset_in_words, Register reg);
 167 
 168   ~StubFrame();
 169 };;
 170 
 171 void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
 172   set_info(name, must_gc_arguments);
 173   enter();
 174 }
 175 
 176 void StubAssembler::epilogue() {
 177   leave();
 178   ret(lr);
 179 }
 180 
 181 #define __ _sasm->
 182 
 183 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {
 184   _sasm = sasm;
 185   __ prologue(name, must_gc_arguments);
 186 }
 187 
 188 // load parameters that were stored with LIR_Assembler::store_parameter
 189 // Note: offsets for store_parameter and load_argument must match
 190 void StubFrame::load_argument(int offset_in_words, Register reg) {
 191   __ load_parameter(offset_in_words, reg);
 192 }
 193 
 194 
 195 StubFrame::~StubFrame() {
 196   __ epilogue();
 197 }
 198 
 199 #undef __
 200 
 201 
 202 // Implementation of Runtime1
 203 
 204 #define __ sasm->
 205 
 206 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
 207 
 208 // Stack layout for saving/restoring  all the registers needed during a runtime
 209 // call (this includes deoptimization)
 210 // Note: note that users of this frame may well have arguments to some runtime
 211 // while these values are on the stack. These positions neglect those arguments
 212 // but the code in save_live_registers will take the argument count into
 213 // account.
 214 //
 215 
 216 enum reg_save_layout {
 217   reg_save_frame_size = 32 /* float */ + 32 /* integer */
 218 };
 219 
 220 // Save off registers which might be killed by calls into the runtime.
 221 // Tries to smart of about FP registers.  In particular we separate
 222 // saving and describing the FPU registers for deoptimization since we
 223 // have to save the FPU registers twice if we describe them.  The
 224 // deopt blob is the only thing which needs to describe FPU registers.
 225 // In all other cases it should be sufficient to simply save their
 226 // current value.
 227 
 228 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
 229 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
 230 static int reg_save_size_in_words;
 231 static int frame_size_in_bytes = -1;
 232 
 233 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
 234   int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
 235   sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
 236   int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
 237   OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
 238 
 239   for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {
 240     Register r = as_Register(i);
 241     if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) {
 242       int sp_offset = cpu_reg_save_offsets[i];
 243       oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 244                                 r->as_VMReg());
 245     }
 246   }
 247 
 248   if (save_fpu_registers) {
 249     for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
 250       FloatRegister r = as_FloatRegister(i);
 251       {
 252         int sp_offset = fpu_reg_save_offsets[i];
 253         oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 254                                   r->as_VMReg());
 255       }
 256     }
 257   }
 258   return oop_map;
 259 }
 260 
 261 static OopMap* save_live_registers(StubAssembler* sasm,
 262                                    bool save_fpu_registers = true) {
 263   __ block_comment("save_live_registers");
 264 
 265   __ push(RegSet::range(r0, r29), sp);         // integer registers except lr & sp
 266 
 267   if (save_fpu_registers) {
 268     for (int i = 31; i>= 0; i -= 4) {
 269       __ sub(sp, sp, 4 * wordSize); // no pre-increment for st1. Emulate it without modifying other registers
 270       __ st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1),
 271           as_FloatRegister(i), __ T1D, Address(sp));
 272     }
 273   } else {
 274     __ add(sp, sp, -32 * wordSize);
 275   }
 276 
 277   return generate_oop_map(sasm, save_fpu_registers);
 278 }
 279 
 280 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
 281   if (restore_fpu_registers) {
 282     for (int i = 0; i < 32; i += 4)
 283       __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
 284           as_FloatRegister(i+3), __ T1D, Address(__ post(sp, 4 * wordSize)));
 285   } else {
 286     __ add(sp, sp, 32 * wordSize);
 287   }
 288 
 289   __ pop(RegSet::range(r0, r29), sp);
 290 }
 291 
 292 static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true)  {
 293 
 294   if (restore_fpu_registers) {
 295     for (int i = 0; i < 32; i += 4)
 296       __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
 297           as_FloatRegister(i+3), __ T1D, Address(__ post(sp, 4 * wordSize)));
 298   } else {
 299     __ add(sp, sp, 32 * wordSize);
 300   }
 301 
 302   __ ldp(zr, r1, Address(__ post(sp, 16)));
 303   __ pop(RegSet::range(r2, r29), sp);
 304 }
 305 
 306 
 307 
 308 void Runtime1::initialize_pd() {
 309   int i;
 310   int sp_offset = 0;
 311 
 312   // all float registers are saved explicitly
 313   assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");
 314   for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
 315     fpu_reg_save_offsets[i] = sp_offset;
 316     sp_offset += 2;   // SP offsets are in halfwords
 317   }
 318 
 319   for (i = 0; i < FrameMap::nof_cpu_regs; i++) {
 320     Register r = as_Register(i);
 321     cpu_reg_save_offsets[i] = sp_offset;
 322     sp_offset += 2;   // SP offsets are in halfwords
 323   }
 324 }
 325 
 326 
 327 // target: the entry point of the method that creates and posts the exception oop
 328 // has_argument: true if the exception needs arguments (passed in rscratch1 and rscratch2)
 329 
 330 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
 331   // make a frame and preserve the caller's caller-save registers
 332   OopMap* oop_map = save_live_registers(sasm);
 333   int call_offset;
 334   if (!has_argument) {
 335     call_offset = __ call_RT(noreg, noreg, target);
 336   } else {
 337     __ mov(c_rarg1, rscratch1);
 338     __ mov(c_rarg2, rscratch2);
 339     call_offset = __ call_RT(noreg, noreg, target);
 340   }
 341   OopMapSet* oop_maps = new OopMapSet();
 342   oop_maps->add_gc_map(call_offset, oop_map);
 343 
 344   __ should_not_reach_here();
 345   return oop_maps;
 346 }
 347 
 348 
 349 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
 350   __ block_comment("generate_handle_exception");
 351 
 352   // incoming parameters
 353   const Register exception_oop = r0;
 354   const Register exception_pc  = r3;
 355   // other registers used in this stub
 356 
 357   // Save registers, if required.
 358   OopMapSet* oop_maps = new OopMapSet();
 359   OopMap* oop_map = NULL;
 360   switch (id) {
 361   case forward_exception_id:
 362     // We're handling an exception in the context of a compiled frame.
 363     // The registers have been saved in the standard places.  Perform
 364     // an exception lookup in the caller and dispatch to the handler
 365     // if found.  Otherwise unwind and dispatch to the callers
 366     // exception handler.
 367     oop_map = generate_oop_map(sasm, 1 /*thread*/);
 368 
 369     // load and clear pending exception oop into r0
 370     __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));
 371     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 372 
 373     // load issuing PC (the return address for this stub) into r3
 374     __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
 375 
 376     // make sure that the vm_results are cleared (may be unnecessary)
 377     __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
 378     __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
 379     break;
 380   case handle_exception_nofpu_id:
 381   case handle_exception_id:
 382     // At this point all registers MAY be live.
 383     oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
 384     break;
 385   case handle_exception_from_callee_id: {
 386     // At this point all registers except exception oop (r0) and
 387     // exception pc (lr) are dead.
 388     const int frame_size = 2 /*fp, return address*/;
 389     oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
 390     sasm->set_frame_size(frame_size);
 391     break;
 392   }
 393   default:
 394     __ should_not_reach_here();
 395     break;
 396   }
 397 
 398   // verify that only r0 and r3 are valid at this time
 399   __ invalidate_registers(false, true, true, false, true, true);
 400   // verify that r0 contains a valid exception
 401   __ verify_not_null_oop(exception_oop);
 402 
 403 #ifdef ASSERT
 404   // check that fields in JavaThread for exception oop and issuing pc are
 405   // empty before writing to them
 406   Label oop_empty;
 407   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 408   __ cbz(rscratch1, oop_empty);
 409   __ stop("exception oop already set");
 410   __ bind(oop_empty);
 411 
 412   Label pc_empty;
 413   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 414   __ cbz(rscratch1, pc_empty);
 415   __ stop("exception pc already set");
 416   __ bind(pc_empty);
 417 #endif
 418 
 419   // save exception oop and issuing pc into JavaThread
 420   // (exception handler will load it from here)
 421   __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));
 422   __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
 423 
 424   // patch throwing pc into return address (has bci & oop map)
 425   __ str(exception_pc, Address(rfp, 1*BytesPerWord));
 426 
 427   // compute the exception handler.
 428   // the exception oop and the throwing pc are read from the fields in JavaThread
 429   int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
 430   oop_maps->add_gc_map(call_offset, oop_map);
 431 
 432   // r0: handler address
 433   //      will be the deopt blob if nmethod was deoptimized while we looked up
 434   //      handler regardless of whether handler existed in the nmethod.
 435 
 436   // only r0 is valid at this time, all other registers have been destroyed by the runtime call
 437   __ invalidate_registers(false, true, true, true, true, true);
 438 
 439   // patch the return address, this stub will directly return to the exception handler
 440   __ str(r0, Address(rfp, 1*BytesPerWord));
 441 
 442   switch (id) {
 443   case forward_exception_id:
 444   case handle_exception_nofpu_id:
 445   case handle_exception_id:
 446     // Restore the registers that were saved at the beginning.
 447     restore_live_registers(sasm, id != handle_exception_nofpu_id);
 448     break;
 449   case handle_exception_from_callee_id:
 450     // Pop the return address.
 451     __ leave();
 452     __ ret(lr);  // jump to exception handler
 453     break;
 454   default:  ShouldNotReachHere();
 455   }
 456 
 457   return oop_maps;
 458 }
 459 
 460 
 461 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
 462   // incoming parameters
 463   const Register exception_oop = r0;
 464   // callee-saved copy of exception_oop during runtime call
 465   const Register exception_oop_callee_saved = r19;
 466   // other registers used in this stub
 467   const Register exception_pc = r3;
 468   const Register handler_addr = r1;
 469 
 470   // verify that only r0, is valid at this time
 471   __ invalidate_registers(false, true, true, true, true, true);
 472 
 473 #ifdef ASSERT
 474   // check that fields in JavaThread for exception oop and issuing pc are empty
 475   Label oop_empty;
 476   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 477   __ cbz(rscratch1, oop_empty);
 478   __ stop("exception oop must be empty");
 479   __ bind(oop_empty);
 480 
 481   Label pc_empty;
 482   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 483   __ cbz(rscratch1, pc_empty);
 484   __ stop("exception pc must be empty");
 485   __ bind(pc_empty);
 486 #endif
 487 
 488   // Save our return address because
 489   // exception_handler_for_return_address will destroy it.  We also
 490   // save exception_oop
 491   __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
 492 
 493   // search the exception handler address of the caller (using the return address)
 494   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr);
 495   // r0: exception handler address of the caller
 496 
 497   // Only R0 is valid at this time; all other registers have been
 498   // destroyed by the call.
 499   __ invalidate_registers(false, true, true, true, false, true);
 500 
 501   // move result of call into correct register
 502   __ mov(handler_addr, r0);
 503 
 504   // get throwing pc (= return address).
 505   // lr has been destroyed by the call
 506   __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
 507   __ mov(r3, lr);
 508 
 509   __ verify_not_null_oop(exception_oop);
 510 
 511   // continue at exception handler (return address removed)
 512   // note: do *not* remove arguments when unwinding the
 513   //       activation since the caller assumes having
 514   //       all arguments on the stack when entering the
 515   //       runtime to determine the exception handler
 516   //       (GC happens at call site with arguments!)
 517   // r0: exception oop
 518   // r3: throwing pc
 519   // r1: exception handler
 520   __ br(handler_addr);
 521 }
 522 
 523 
 524 
 525 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
 526   // use the maximum number of runtime-arguments here because it is difficult to
 527   // distinguish each RT-Call.
 528   // Note: This number affects also the RT-Call in generate_handle_exception because
 529   //       the oop-map is shared for all calls.
 530   DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
 531   assert(deopt_blob != NULL, "deoptimization blob must have been created");
 532 
 533   OopMap* oop_map = save_live_registers(sasm);
 534 
 535   __ mov(c_rarg0, rthread);
 536   Label retaddr;
 537   __ set_last_Java_frame(sp, rfp, retaddr, rscratch1);
 538   // do the call
 539   __ lea(rscratch1, RuntimeAddress(target));
 540   __ blrt(rscratch1, 1, 0, 1);
 541   __ bind(retaddr);
 542   OopMapSet* oop_maps = new OopMapSet();
 543   oop_maps->add_gc_map(__ offset(), oop_map);
 544   // verify callee-saved register
 545 #ifdef ASSERT
 546   { Label L;
 547     __ get_thread(rscratch1);
 548     __ cmp(rthread, rscratch1);
 549     __ br(Assembler::EQ, L);
 550     __ stop("StubAssembler::call_RT: rthread not callee saved?");
 551     __ bind(L);
 552   }
 553 #endif
 554   __ reset_last_Java_frame(true);
 555   __ maybe_isb();
 556 
 557   // check for pending exceptions
 558   { Label L;
 559     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 560     __ cbz(rscratch1, L);
 561     // exception pending => remove activation and forward to exception handler
 562 
 563     { Label L1;
 564       __ cbnz(r0, L1);                                  // have we deoptimized?
 565       __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 566       __ bind(L1);
 567     }
 568 
 569     // the deopt blob expects exceptions in the special fields of
 570     // JavaThread, so copy and clear pending exception.
 571 
 572     // load and clear pending exception
 573     __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 574     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 575 
 576     // check that there is really a valid exception
 577     __ verify_not_null_oop(r0);
 578 
 579     // load throwing pc: this is the return address of the stub
 580     __ mov(r3, lr);
 581 
 582 #ifdef ASSERT
 583     // check that fields in JavaThread for exception oop and issuing pc are empty
 584     Label oop_empty;
 585     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 586     __ cbz(rscratch1, oop_empty);
 587     __ stop("exception oop must be empty");
 588     __ bind(oop_empty);
 589 
 590     Label pc_empty;
 591     __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 592     __ cbz(rscratch1, pc_empty);
 593     __ stop("exception pc must be empty");
 594     __ bind(pc_empty);
 595 #endif
 596 
 597     // store exception oop and throwing pc to JavaThread
 598     __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
 599     __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
 600 
 601     restore_live_registers(sasm);
 602 
 603     __ leave();
 604 
 605     // Forward the exception directly to deopt blob. We can blow no
 606     // registers and must leave throwing pc on the stack.  A patch may
 607     // have values live in registers so the entry point with the
 608     // exception in tls.
 609     __ far_jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls()));
 610 
 611     __ bind(L);
 612   }
 613 
 614 
 615   // Runtime will return true if the nmethod has been deoptimized during
 616   // the patching process. In that case we must do a deopt reexecute instead.
 617 
 618   Label reexecuteEntry, cont;
 619 
 620   __ cbz(r0, cont);                                 // have we deoptimized?
 621 
 622   // Will reexecute. Proper return address is already on the stack we just restore
 623   // registers, pop all of our frame but the return address and jump to the deopt blob
 624   restore_live_registers(sasm);
 625   __ leave();
 626   __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
 627 
 628   __ bind(cont);
 629   restore_live_registers(sasm);
 630   __ leave();
 631   __ ret(lr);
 632 
 633   return oop_maps;
 634 }
 635 
 636 
 637 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
 638 
 639   const Register exception_oop = r0;
 640   const Register exception_pc  = r3;
 641 
 642   // for better readability
 643   const bool must_gc_arguments = true;
 644   const bool dont_gc_arguments = false;
 645 
 646   // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
 647   bool save_fpu_registers = true;
 648 
 649   // stub code & info for the different stubs
 650   OopMapSet* oop_maps = NULL;
 651   OopMap* oop_map = NULL;
 652   switch (id) {
 653     {
 654     case forward_exception_id:
 655       {
 656         oop_maps = generate_handle_exception(id, sasm);
 657         __ leave();
 658         __ ret(lr);
 659       }
 660       break;
 661 
 662     case throw_div0_exception_id:
 663       { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments);
 664         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
 665       }
 666       break;
 667 
 668     case throw_null_pointer_exception_id:
 669       { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments);
 670         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
 671       }
 672       break;
 673 
 674     case new_instance_id:
 675     case fast_new_instance_id:
 676     case fast_new_instance_init_check_id:
 677       {
 678         Register klass = r3; // Incoming
 679         Register obj   = r0; // Result
 680 
 681         if (id == new_instance_id) {
 682           __ set_info("new_instance", dont_gc_arguments);
 683         } else if (id == fast_new_instance_id) {
 684           __ set_info("fast new_instance", dont_gc_arguments);
 685         } else {
 686           assert(id == fast_new_instance_init_check_id, "bad StubID");
 687           __ set_info("fast new_instance init check", dont_gc_arguments);
 688         }
 689 



 690         if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
 691             UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
 692           Label slow_path;
 693           Register obj_size = r2;
 694           Register t1       = r19;
 695           Register t2       = r4;
 696           assert_different_registers(klass, obj, obj_size, t1, t2);
 697 
 698           __ stp(r19, zr, Address(__ pre(sp, -2 * wordSize)));
 699 
 700           if (id == fast_new_instance_init_check_id) {
 701             // make sure the klass is initialized
 702             __ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset()));
 703             __ cmpw(rscratch1, InstanceKlass::fully_initialized);
 704             __ br(Assembler::NE, slow_path);
 705           }
 706 
 707 #ifdef ASSERT
 708           // assert object can be fast path allocated
 709           {
 710             Label ok, not_ok;
 711             __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 712             __ cmp(obj_size, 0u);
 713             __ br(Assembler::LE, not_ok);  // make sure it's an instance (LH > 0)
 714             __ tstw(obj_size, Klass::_lh_instance_slow_path_bit);
 715             __ br(Assembler::EQ, ok);
 716             __ bind(not_ok);
 717             __ stop("assert(can be fast path allocated)");
 718             __ should_not_reach_here();
 719             __ bind(ok);
 720           }
 721 #endif // ASSERT
 722 
 723           // get the instance size (size is postive so movl is fine for 64bit)
 724           __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 725 
 726           __ eden_allocate(obj, obj_size, 0, t1, slow_path);
 727 
 728           __ initialize_object(obj, klass, obj_size, 0, t1, t2, /* is_tlab_allocated */ false);
 729           __ verify_oop(obj);
 730           __ ldp(r19, zr, Address(__ post(sp, 2 * wordSize)));
 731           __ ret(lr);
 732 
 733           __ bind(slow_path);
 734           __ ldp(r19, zr, Address(__ post(sp, 2 * wordSize)));
 735         }
 736 
 737         __ enter();
 738         OopMap* map = save_live_registers(sasm);
 739         int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
 740         oop_maps = new OopMapSet();
 741         oop_maps->add_gc_map(call_offset, map);
 742         restore_live_registers_except_r0(sasm);
 743         __ verify_oop(obj);
 744         __ leave();
 745         __ ret(lr);
 746 
 747         // r0,: new instance
 748       }
 749 
 750       break;
 751 
 752     case counter_overflow_id:
 753       {
 754         Register bci = r0, method = r1;
 755         __ enter();
 756         OopMap* map = save_live_registers(sasm);
 757         // Retrieve bci
 758         __ ldrw(bci, Address(rfp, 2*BytesPerWord));
 759         // And a pointer to the Method*
 760         __ ldr(method, Address(rfp, 3*BytesPerWord));
 761         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
 762         oop_maps = new OopMapSet();
 763         oop_maps->add_gc_map(call_offset, map);
 764         restore_live_registers(sasm);
 765         __ leave();
 766         __ ret(lr);
 767       }
 768       break;
 769 
 770     case new_type_array_id:
 771     case new_object_array_id:
 772       {
 773         Register length   = r19; // Incoming
 774         Register klass    = r3; // Incoming
 775         Register obj      = r0; // Result
 776 
 777         if (id == new_type_array_id) {
 778           __ set_info("new_type_array", dont_gc_arguments);
 779         } else {
 780           __ set_info("new_object_array", dont_gc_arguments);
 781         }
 782 
 783 #ifdef ASSERT
 784         // assert object type is really an array of the proper kind
 785         {
 786           Label ok;
 787           Register t0 = obj;
 788           __ ldrw(t0, Address(klass, Klass::layout_helper_offset()));
 789           __ asrw(t0, t0, Klass::_lh_array_tag_shift);
 790           int tag = ((id == new_type_array_id)
 791                      ? Klass::_lh_array_tag_type_value
 792                      : Klass::_lh_array_tag_obj_value);
 793           __ mov(rscratch1, tag);
 794           __ cmpw(t0, rscratch1);
 795           __ br(Assembler::EQ, ok);
 796           __ stop("assert(is an array klass)");
 797           __ should_not_reach_here();
 798           __ bind(ok);
 799         }
 800 #endif // ASSERT
 801 
 802         if (UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {



 803           Register arr_size = r4;
 804           Register t1       = r2;
 805           Register t2       = r5;
 806           Label slow_path;
 807           assert_different_registers(length, klass, obj, arr_size, t1, t2);
 808 
 809           // check that array length is small enough for fast path.
 810           __ mov(rscratch1, C1_MacroAssembler::max_array_allocation_length);
 811           __ cmpw(length, rscratch1);
 812           __ br(Assembler::HI, slow_path);
 813 
 814           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
 815           // since size is positive ldrw does right thing on 64bit
 816           __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
 817           // since size is positive movw does right thing on 64bit
 818           __ movw(arr_size, length);
 819           __ lslvw(arr_size, length, t1);
 820           __ ubfx(t1, t1, Klass::_lh_header_size_shift,
 821                   exact_log2(Klass::_lh_header_size_mask + 1));
 822           __ add(arr_size, arr_size, t1);
 823           __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
 824           __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 825 
 826           __ eden_allocate(obj, arr_size, 0, t1, slow_path);  // preserves arr_size
 827 
 828           __ initialize_header(obj, klass, length, t1, t2);
 829           __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
 830           assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
 831           assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
 832           __ andr(t1, t1, Klass::_lh_header_size_mask);
 833           __ sub(arr_size, arr_size, t1);  // body length
 834           __ add(t1, t1, obj);       // body start
 835           __ initialize_body(t1, arr_size, 0, t2);
 836           __ verify_oop(obj);
 837 
 838           __ ret(lr);
 839 
 840           __ bind(slow_path);
 841         }
 842 
 843         __ enter();
 844         OopMap* map = save_live_registers(sasm);
 845         int call_offset;
 846         if (id == new_type_array_id) {
 847           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
 848         } else {
 849           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
 850         }
 851 
 852         oop_maps = new OopMapSet();
 853         oop_maps->add_gc_map(call_offset, map);
 854         restore_live_registers_except_r0(sasm);
 855 
 856         __ verify_oop(obj);
 857         __ leave();
 858         __ ret(lr);
 859 
 860         // r0: new array
 861       }
 862       break;
 863 
 864     case new_multi_array_id:
 865       { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
 866         // r0,: klass
 867         // r19,: rank
 868         // r2: address of 1st dimension
 869         OopMap* map = save_live_registers(sasm);
 870         __ mov(c_rarg1, r0);
 871         __ mov(c_rarg3, r2);
 872         __ mov(c_rarg2, r19);
 873         int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3);
 874 
 875         oop_maps = new OopMapSet();
 876         oop_maps->add_gc_map(call_offset, map);
 877         restore_live_registers_except_r0(sasm);
 878 
 879         // r0,: new multi array
 880         __ verify_oop(r0);
 881       }
 882       break;
 883 
 884     case register_finalizer_id:
 885       {
 886         __ set_info("register_finalizer", dont_gc_arguments);
 887 
 888         // This is called via call_runtime so the arguments
 889         // will be place in C abi locations
 890 
 891         __ verify_oop(c_rarg0);
 892 
 893         // load the klass and check the has finalizer flag
 894         Label register_finalizer;
 895         Register t = r5;
 896         __ load_klass(t, r0);
 897         __ ldrw(t, Address(t, Klass::access_flags_offset()));
 898         __ tbnz(t, exact_log2(JVM_ACC_HAS_FINALIZER), register_finalizer);
 899         __ ret(lr);
 900 
 901         __ bind(register_finalizer);
 902         __ enter();
 903         OopMap* oop_map = save_live_registers(sasm);
 904         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0);
 905         oop_maps = new OopMapSet();
 906         oop_maps->add_gc_map(call_offset, oop_map);
 907 
 908         // Now restore all the live registers
 909         restore_live_registers(sasm);
 910 
 911         __ leave();
 912         __ ret(lr);
 913       }
 914       break;
 915 
 916     case throw_class_cast_exception_id:
 917       { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments);
 918         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
 919       }
 920       break;
 921 
 922     case throw_incompatible_class_change_error_id:
 923       { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments);
 924         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
 925       }
 926       break;
 927 
 928     case slow_subtype_check_id:
 929       {
 930         // Typical calling sequence:
 931         // __ push(klass_RInfo);  // object klass or other subclass
 932         // __ push(sup_k_RInfo);  // array element klass or other superclass
 933         // __ bl(slow_subtype_check);
 934         // Note that the subclass is pushed first, and is therefore deepest.
 935         enum layout {
 936           r0_off, r0_off_hi,
 937           r2_off, r2_off_hi,
 938           r4_off, r4_off_hi,
 939           r5_off, r5_off_hi,
 940           sup_k_off, sup_k_off_hi,
 941           klass_off, klass_off_hi,
 942           framesize,
 943           result_off = sup_k_off
 944         };
 945 
 946         __ set_info("slow_subtype_check", dont_gc_arguments);
 947         __ push(RegSet::of(r0, r2, r4, r5), sp);
 948 
 949         // This is called by pushing args and not with C abi
 950         // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
 951         // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
 952 
 953         __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
 954 
 955         Label miss;
 956         __ check_klass_subtype_slow_path(r4, r0, r2, r5, NULL, &miss);
 957 
 958         // fallthrough on success:
 959         __ mov(rscratch1, 1);
 960         __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
 961         __ pop(RegSet::of(r0, r2, r4, r5), sp);
 962         __ ret(lr);
 963 
 964         __ bind(miss);
 965         __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
 966         __ pop(RegSet::of(r0, r2, r4, r5), sp);
 967         __ ret(lr);
 968       }
 969       break;
 970 
 971     case monitorenter_nofpu_id:
 972       save_fpu_registers = false;
 973       // fall through
 974     case monitorenter_id:
 975       {
 976         StubFrame f(sasm, "monitorenter", dont_gc_arguments);
 977         OopMap* map = save_live_registers(sasm, save_fpu_registers);
 978 
 979         // Called with store_parameter and not C abi
 980 
 981         f.load_argument(1, r0); // r0,: object
 982         f.load_argument(0, r1); // r1,: lock address
 983 
 984         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1);
 985 
 986         oop_maps = new OopMapSet();
 987         oop_maps->add_gc_map(call_offset, map);
 988         restore_live_registers(sasm, save_fpu_registers);
 989       }
 990       break;
 991 
 992     case monitorexit_nofpu_id:
 993       save_fpu_registers = false;
 994       // fall through
 995     case monitorexit_id:
 996       {
 997         StubFrame f(sasm, "monitorexit", dont_gc_arguments);
 998         OopMap* map = save_live_registers(sasm, save_fpu_registers);
 999 
1000         // Called with store_parameter and not C abi
1001 
1002         f.load_argument(0, r0); // r0,: lock address
1003 
1004         // note: really a leaf routine but must setup last java sp
1005         //       => use call_RT for now (speed can be improved by
1006         //       doing last java sp setup manually)
1007         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0);
1008 
1009         oop_maps = new OopMapSet();
1010         oop_maps->add_gc_map(call_offset, map);
1011         restore_live_registers(sasm, save_fpu_registers);
1012       }
1013       break;
1014 
1015     case deoptimize_id:
1016       {
1017         StubFrame f(sasm, "deoptimize", dont_gc_arguments);
1018         OopMap* oop_map = save_live_registers(sasm);
1019         f.load_argument(0, c_rarg1);
1020         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1);
1021 
1022         oop_maps = new OopMapSet();
1023         oop_maps->add_gc_map(call_offset, oop_map);
1024         restore_live_registers(sasm);
1025         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1026         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1027         __ leave();
1028         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1029       }
1030       break;
1031 
1032     case throw_range_check_failed_id:
1033       { StubFrame f(sasm, "range_check_failed", dont_gc_arguments);
1034         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1035       }
1036       break;
1037 
1038     case unwind_exception_id:
1039       { __ set_info("unwind_exception", dont_gc_arguments);
1040         // note: no stubframe since we are about to leave the current
1041         //       activation and we are calling a leaf VM function only.
1042         generate_unwind_exception(sasm);
1043       }
1044       break;
1045 
1046     case access_field_patching_id:
1047       { StubFrame f(sasm, "access_field_patching", dont_gc_arguments);
1048         // we should set up register map
1049         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1050       }
1051       break;
1052 
1053     case load_klass_patching_id:
1054       { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments);
1055         // we should set up register map
1056         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1057       }
1058       break;
1059 
1060     case load_mirror_patching_id:
1061       { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments);
1062         // we should set up register map
1063         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1064       }
1065       break;
1066 
1067     case load_appendix_patching_id:
1068       { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments);
1069         // we should set up register map
1070         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1071       }
1072       break;
1073 
1074     case handle_exception_nofpu_id:
1075     case handle_exception_id:
1076       { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1077         oop_maps = generate_handle_exception(id, sasm);
1078       }
1079       break;
1080 
1081     case handle_exception_from_callee_id:
1082       { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1083         oop_maps = generate_handle_exception(id, sasm);
1084       }
1085       break;
1086 
1087     case throw_index_exception_id:
1088       { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments);
1089         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1090       }
1091       break;
1092 
1093     case throw_array_store_exception_id:
1094       { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments);
1095         // tos + 0: link
1096         //     + 1: return address
1097         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1098       }
1099       break;
1100 
1101     case predicate_failed_trap_id:
1102       {
1103         StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1104 
1105         OopMap* map = save_live_registers(sasm);
1106 
1107         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1108         oop_maps = new OopMapSet();
1109         oop_maps->add_gc_map(call_offset, map);
1110         restore_live_registers(sasm);
1111         __ leave();
1112         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1113         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1114 
1115         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1116       }
1117       break;
1118 
1119 
1120     default:
1121       { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
1122         __ mov(r0, (int)id);
1123         __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1124         __ should_not_reach_here();
1125       }
1126       break;
1127     }
1128   }
1129   return oop_maps;
1130 }
1131 
1132 #undef __
1133 
1134 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }
--- EOF ---