1 /*
   2  * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "code/debugInfoRec.hpp"
  29 #include "code/icBuffer.hpp"
  30 #include "code/vtableStubs.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "oops/compiledICHolder.hpp"
  33 #include "prims/jvmtiRedefineClassesTrace.hpp"
  34 #include "runtime/sharedRuntime.hpp"
  35 #include "runtime/vframeArray.hpp"
  36 #include "vmreg_x86.inline.hpp"
  37 #ifdef COMPILER1
  38 #include "c1/c1_Runtime1.hpp"
  39 #endif
  40 #ifdef COMPILER2
  41 #include "opto/runtime.hpp"
  42 #endif
  43 
  44 #define __ masm->
  45 
  46 const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;
  47 
  48 class RegisterSaver {
  49   // Capture info about frame layout
  50 #define DEF_XMM_OFFS(regnum) xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off
  51   enum layout {
  52                 fpu_state_off = 0,
  53                 fpu_state_end = fpu_state_off+FPUStateSizeInWords,
  54                 st0_off, st0H_off,
  55                 st1_off, st1H_off,
  56                 st2_off, st2H_off,
  57                 st3_off, st3H_off,
  58                 st4_off, st4H_off,
  59                 st5_off, st5H_off,
  60                 st6_off, st6H_off,
  61                 st7_off, st7H_off,
  62                 xmm_off,
  63                 DEF_XMM_OFFS(0),
  64                 DEF_XMM_OFFS(1),
  65                 DEF_XMM_OFFS(2),
  66                 DEF_XMM_OFFS(3),
  67                 DEF_XMM_OFFS(4),
  68                 DEF_XMM_OFFS(5),
  69                 DEF_XMM_OFFS(6),
  70                 DEF_XMM_OFFS(7),
  71                 flags_off = xmm7_off + 16/BytesPerInt + 1, // 16-byte stack alignment fill word
  72                 rdi_off,
  73                 rsi_off,
  74                 ignore_off,  // extra copy of rbp,
  75                 rsp_off,
  76                 rbx_off,
  77                 rdx_off,
  78                 rcx_off,
  79                 rax_off,
  80                 // The frame sender code expects that rbp will be in the "natural" place and
  81                 // will override any oopMap setting for it. We must therefore force the layout
  82                 // so that it agrees with the frame sender code.
  83                 rbp_off,
  84                 return_off,      // slot for return address
  85                 reg_save_size };
  86   enum { FPU_regs_live = flags_off - fpu_state_end };
  87 
  88   public:
  89 
  90   static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words,
  91                                      int* total_frame_words, bool verify_fpu = true, bool save_vectors = false);
  92   static void restore_live_registers(MacroAssembler* masm, bool restore_vectors = false);
  93 
  94   static int rax_offset() { return rax_off; }
  95   static int rbx_offset() { return rbx_off; }
  96 
  97   // Offsets into the register save area
  98   // Used by deoptimization when it is managing result register
  99   // values on its own
 100 
 101   static int raxOffset(void) { return rax_off; }
 102   static int rdxOffset(void) { return rdx_off; }
 103   static int rbxOffset(void) { return rbx_off; }
 104   static int xmm0Offset(void) { return xmm0_off; }
 105   // This really returns a slot in the fp save area, which one is not important
 106   static int fpResultOffset(void) { return st0_off; }
 107 
 108   // During deoptimization only the result register need to be restored
 109   // all the other values have already been extracted.
 110 
 111   static void restore_result_registers(MacroAssembler* masm);
 112 
 113 };
 114 
 115 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words,
 116                                            int* total_frame_words, bool verify_fpu, bool save_vectors) {
 117   int vect_words = 0;
 118   int num_xmm_regs = XMMRegisterImpl::number_of_registers;
 119 #ifdef COMPILER2
 120   if (save_vectors) {
 121     assert(UseAVX > 0, "512bit vectors are supported only with EVEX");
 122     assert(MaxVectorSize == 64, "only 512bit vectors are supported now");
 123     // Save upper half of ZMM/YMM registers :
 124     vect_words = 8 * 16 / wordSize;
 125     additional_frame_words += vect_words;
 126   }
 127 #else
 128   assert(!save_vectors, "vectors are generated only by C2");
 129 #endif
 130   int frame_size_in_bytes = (reg_save_size + additional_frame_words) * wordSize;
 131   int frame_words = frame_size_in_bytes / wordSize;
 132   *total_frame_words = frame_words;
 133 
 134   assert(FPUStateSizeInWords == 27, "update stack layout");
 135 
 136   // save registers, fpu state, and flags
 137   // We assume caller has already has return address slot on the stack
 138   // We push epb twice in this sequence because we want the real rbp,
 139   // to be under the return like a normal enter and we want to use pusha
 140   // We push by hand instead of pusing push
 141   __ enter();
 142   __ pusha();
 143   __ pushf();
 144   __ subptr(rsp,FPU_regs_live*wordSize); // Push FPU registers space
 145   __ push_FPU_state();          // Save FPU state & init
 146 
 147   if (verify_fpu) {
 148     // Some stubs may have non standard FPU control word settings so
 149     // only check and reset the value when it required to be the
 150     // standard value.  The safepoint blob in particular can be used
 151     // in methods which are using the 24 bit control word for
 152     // optimized float math.
 153 
 154 #ifdef ASSERT
 155     // Make sure the control word has the expected value
 156     Label ok;
 157     __ cmpw(Address(rsp, 0), StubRoutines::fpu_cntrl_wrd_std());
 158     __ jccb(Assembler::equal, ok);
 159     __ stop("corrupted control word detected");
 160     __ bind(ok);
 161 #endif
 162 
 163     // Reset the control word to guard against exceptions being unmasked
 164     // since fstp_d can cause FPU stack underflow exceptions.  Write it
 165     // into the on stack copy and then reload that to make sure that the
 166     // current and future values are correct.
 167     __ movw(Address(rsp, 0), StubRoutines::fpu_cntrl_wrd_std());
 168   }
 169 
 170   __ frstor(Address(rsp, 0));
 171   if (!verify_fpu) {
 172     // Set the control word so that exceptions are masked for the
 173     // following code.
 174     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 175   }
 176 
 177   int off = st0_off;
 178   int delta = st1_off - off;
 179 
 180   // Save the FPU registers in de-opt-able form
 181   for (int n = 0; n < FloatRegisterImpl::number_of_registers; n++) {
 182     __ fstp_d(Address(rsp, off*wordSize));
 183     off += delta;
 184   }
 185 
 186   off = xmm0_off;
 187   delta = xmm1_off - off;
 188   if(UseSSE == 1) {           // Save the XMM state
 189     for (int n = 0; n < num_xmm_regs; n++) {
 190       __ movflt(Address(rsp, off*wordSize), as_XMMRegister(n));
 191       off += delta;
 192     }
 193   } else if(UseSSE >= 2) {
 194     // Save whole 128bit (16 bytes) XMM regiters
 195     for (int n = 0; n < num_xmm_regs; n++) {
 196       __ movdqu(Address(rsp, off*wordSize), as_XMMRegister(n));
 197       off += delta;
 198     }
 199   }
 200 
 201   if (save_vectors) {
 202     assert(vect_words*wordSize == 128, "");
 203     __ subptr(rsp, 128); // Save upper half of YMM registes
 204     for (int n = 0; n < num_xmm_regs; n++) {
 205       __ vextractf128h(Address(rsp, n*16), as_XMMRegister(n));
 206     }
 207     if (UseAVX > 2) {
 208       __ subptr(rsp, 256); // Save upper half of ZMM registes
 209       for (int n = 0; n < num_xmm_regs; n++) {
 210         __ vextractf64x4h(Address(rsp, n*32), as_XMMRegister(n), 1);
 211       }
 212     }
 213   }
 214 
 215   // Set an oopmap for the call site.  This oopmap will map all
 216   // oop-registers and debug-info registers as callee-saved.  This
 217   // will allow deoptimization at this safepoint to find all possible
 218   // debug-info recordings, as well as let GC find all oops.
 219 
 220   OopMapSet *oop_maps = new OopMapSet();
 221   OopMap* map =  new OopMap( frame_words, 0 );
 222 
 223 #define STACK_OFFSET(x) VMRegImpl::stack2reg((x) + additional_frame_words)
 224 #define NEXTREG(x) (x)->as_VMReg()->next()
 225 
 226   map->set_callee_saved(STACK_OFFSET(rax_off), rax->as_VMReg());
 227   map->set_callee_saved(STACK_OFFSET(rcx_off), rcx->as_VMReg());
 228   map->set_callee_saved(STACK_OFFSET(rdx_off), rdx->as_VMReg());
 229   map->set_callee_saved(STACK_OFFSET(rbx_off), rbx->as_VMReg());
 230   // rbp, location is known implicitly, no oopMap
 231   map->set_callee_saved(STACK_OFFSET(rsi_off), rsi->as_VMReg());
 232   map->set_callee_saved(STACK_OFFSET(rdi_off), rdi->as_VMReg());
 233   // %%% This is really a waste but we'll keep things as they were for now for the upper component
 234   off = st0_off;
 235   delta = st1_off - off;
 236   for (int n = 0; n < FloatRegisterImpl::number_of_registers; n++) {
 237     FloatRegister freg_name = as_FloatRegister(n);
 238     map->set_callee_saved(STACK_OFFSET(off), freg_name->as_VMReg());
 239     map->set_callee_saved(STACK_OFFSET(off+1), NEXTREG(freg_name));
 240     off += delta;
 241   }
 242   off = xmm0_off;
 243   delta = xmm1_off - off;
 244   for (int n = 0; n < num_xmm_regs; n++) {
 245     XMMRegister xmm_name = as_XMMRegister(n);
 246     map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg());
 247     map->set_callee_saved(STACK_OFFSET(off+1), NEXTREG(xmm_name));
 248     off += delta;
 249   }
 250 #undef NEXTREG
 251 #undef STACK_OFFSET
 252 
 253   return map;
 254 }
 255 
 256 void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {
 257   int num_xmm_regs = XMMRegisterImpl::number_of_registers;
 258   // Recover XMM & FPU state
 259   int additional_frame_bytes = 0;
 260 #ifdef COMPILER2
 261   if (restore_vectors) {
 262     assert(UseAVX > 0, "512bit vectors are supported only with EVEX");
 263     assert(MaxVectorSize == 64, "only 512bit vectors are supported now");
 264     additional_frame_bytes = 128;
 265   }
 266 #else
 267   assert(!restore_vectors, "vectors are generated only by C2");
 268 #endif
 269 
 270   if (restore_vectors) {
 271     assert(additional_frame_bytes == 128, "");
 272     if (UseAVX > 2) {
 273       // Restore upper half of ZMM registers.
 274       for (int n = 0; n < num_xmm_regs; n++) {
 275         __ vinsertf64x4h(as_XMMRegister(n), Address(rsp, n*32), 1);
 276       }
 277       __ addptr(rsp, additional_frame_bytes*2); // Save upper half of ZMM registes
 278     }
 279     // Restore upper half of YMM registes.
 280     for (int n = 0; n < num_xmm_regs; n++) {
 281       __ vinsertf128h(as_XMMRegister(n), Address(rsp, n*16));
 282     }
 283     __ addptr(rsp, additional_frame_bytes); // Save upper half of YMM registes
 284   }
 285 
 286   int off = xmm0_off;
 287   int delta = xmm1_off - off;
 288 
 289   if (UseSSE == 1) {
 290     for (int n = 0; n < num_xmm_regs; n++) {
 291       __ movflt(as_XMMRegister(n), Address(rsp, off*wordSize));
 292       off += delta;
 293     }
 294   } else if (UseSSE >= 2) {
 295     // additional_frame_bytes only populated for the restore_vector case, else it is 0
 296     for (int n = 0; n < num_xmm_regs; n++) {
 297       __ movdqu(as_XMMRegister(n), Address(rsp, off*wordSize+additional_frame_bytes));
 298       off += delta;
 299     }
 300   }
 301 
 302   __ pop_FPU_state();
 303   __ addptr(rsp, FPU_regs_live*wordSize); // Pop FPU registers
 304 
 305   __ popf();
 306   __ popa();
 307   // Get the rbp, described implicitly by the frame sender code (no oopMap)
 308   __ pop(rbp);
 309 
 310 }
 311 
 312 void RegisterSaver::restore_result_registers(MacroAssembler* masm) {
 313 
 314   // Just restore result register. Only used by deoptimization. By
 315   // now any callee save register that needs to be restore to a c2
 316   // caller of the deoptee has been extracted into the vframeArray
 317   // and will be stuffed into the c2i adapter we create for later
 318   // restoration so only result registers need to be restored here.
 319   //
 320 
 321   __ frstor(Address(rsp, 0));      // Restore fpu state
 322 
 323   // Recover XMM & FPU state
 324   if( UseSSE == 1 ) {
 325     __ movflt(xmm0, Address(rsp, xmm0_off*wordSize));
 326   } else if( UseSSE >= 2 ) {
 327     __ movdbl(xmm0, Address(rsp, xmm0_off*wordSize));
 328   }
 329   __ movptr(rax, Address(rsp, rax_off*wordSize));
 330   __ movptr(rdx, Address(rsp, rdx_off*wordSize));
 331   // Pop all of the register save are off the stack except the return address
 332   __ addptr(rsp, return_off * wordSize);
 333 }
 334 
 335 // Is vector's size (in bytes) bigger than a size saved by default?
 336 // 16 bytes XMM registers are saved by default using SSE2 movdqu instructions.
 337 // Note, MaxVectorSize == 0 with UseSSE < 2 and vectors are not generated.
 338 bool SharedRuntime::is_wide_vector(int size) {
 339   return size > 16;
 340 }
 341 
 342 // The java_calling_convention describes stack locations as ideal slots on
 343 // a frame with no abi restrictions. Since we must observe abi restrictions
 344 // (like the placement of the register window) the slots must be biased by
 345 // the following value.
 346 static int reg2offset_in(VMReg r) {
 347   // Account for saved rbp, and return address
 348   // This should really be in_preserve_stack_slots
 349   return (r->reg2stack() + 2) * VMRegImpl::stack_slot_size;
 350 }
 351 
 352 static int reg2offset_out(VMReg r) {
 353   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 354 }
 355 
 356 // ---------------------------------------------------------------------------
 357 // Read the array of BasicTypes from a signature, and compute where the
 358 // arguments should go.  Values in the VMRegPair regs array refer to 4-byte
 359 // quantities.  Values less than SharedInfo::stack0 are registers, those above
 360 // refer to 4-byte stack slots.  All stack slots are based off of the stack pointer
 361 // as framesizes are fixed.
 362 // VMRegImpl::stack0 refers to the first slot 0(sp).
 363 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register
 364 // up to RegisterImpl::number_of_registers) are the 32-bit
 365 // integer registers.
 366 
 367 // Pass first two oop/int args in registers ECX and EDX.
 368 // Pass first two float/double args in registers XMM0 and XMM1.
 369 // Doubles have precedence, so if you pass a mix of floats and doubles
 370 // the doubles will grab the registers before the floats will.
 371 
 372 // Note: the INPUTS in sig_bt are in units of Java argument words, which are
 373 // either 32-bit or 64-bit depending on the build.  The OUTPUTS are in 32-bit
 374 // units regardless of build. Of course for i486 there is no 64 bit build
 375 
 376 
 377 // ---------------------------------------------------------------------------
 378 // The compiled Java calling convention.
 379 // Pass first two oop/int args in registers ECX and EDX.
 380 // Pass first two float/double args in registers XMM0 and XMM1.
 381 // Doubles have precedence, so if you pass a mix of floats and doubles
 382 // the doubles will grab the registers before the floats will.
 383 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
 384                                            VMRegPair *regs,
 385                                            int total_args_passed,
 386                                            int is_outgoing) {
 387   uint    stack = 0;          // Starting stack position for args on stack
 388 
 389 
 390   // Pass first two oop/int args in registers ECX and EDX.
 391   uint reg_arg0 = 9999;
 392   uint reg_arg1 = 9999;
 393 
 394   // Pass first two float/double args in registers XMM0 and XMM1.
 395   // Doubles have precedence, so if you pass a mix of floats and doubles
 396   // the doubles will grab the registers before the floats will.
 397   // CNC - TURNED OFF FOR non-SSE.
 398   //       On Intel we have to round all doubles (and most floats) at
 399   //       call sites by storing to the stack in any case.
 400   // UseSSE=0 ==> Don't Use ==> 9999+0
 401   // UseSSE=1 ==> Floats only ==> 9999+1
 402   // UseSSE>=2 ==> Floats or doubles ==> 9999+2
 403   enum { fltarg_dontuse = 9999+0, fltarg_float_only = 9999+1, fltarg_flt_dbl = 9999+2 };
 404   uint fargs = (UseSSE>=2) ? 2 : UseSSE;
 405   uint freg_arg0 = 9999+fargs;
 406   uint freg_arg1 = 9999+fargs;
 407 
 408   // Pass doubles & longs aligned on the stack.  First count stack slots for doubles
 409   int i;
 410   for( i = 0; i < total_args_passed; i++) {
 411     if( sig_bt[i] == T_DOUBLE ) {
 412       // first 2 doubles go in registers
 413       if( freg_arg0 == fltarg_flt_dbl ) freg_arg0 = i;
 414       else if( freg_arg1 == fltarg_flt_dbl ) freg_arg1 = i;
 415       else // Else double is passed low on the stack to be aligned.
 416         stack += 2;
 417     } else if( sig_bt[i] == T_LONG ) {
 418       stack += 2;
 419     }
 420   }
 421   int dstack = 0;             // Separate counter for placing doubles
 422 
 423   // Now pick where all else goes.
 424   for( i = 0; i < total_args_passed; i++) {
 425     // From the type and the argument number (count) compute the location
 426     switch( sig_bt[i] ) {
 427     case T_SHORT:
 428     case T_CHAR:
 429     case T_BYTE:
 430     case T_BOOLEAN:
 431     case T_INT:
 432     case T_ARRAY:
 433     case T_OBJECT:
 434     case T_ADDRESS:
 435       if( reg_arg0 == 9999 )  {
 436         reg_arg0 = i;
 437         regs[i].set1(rcx->as_VMReg());
 438       } else if( reg_arg1 == 9999 )  {
 439         reg_arg1 = i;
 440         regs[i].set1(rdx->as_VMReg());
 441       } else {
 442         regs[i].set1(VMRegImpl::stack2reg(stack++));
 443       }
 444       break;
 445     case T_FLOAT:
 446       if( freg_arg0 == fltarg_flt_dbl || freg_arg0 == fltarg_float_only ) {
 447         freg_arg0 = i;
 448         regs[i].set1(xmm0->as_VMReg());
 449       } else if( freg_arg1 == fltarg_flt_dbl || freg_arg1 == fltarg_float_only ) {
 450         freg_arg1 = i;
 451         regs[i].set1(xmm1->as_VMReg());
 452       } else {
 453         regs[i].set1(VMRegImpl::stack2reg(stack++));
 454       }
 455       break;
 456     case T_LONG:
 457       assert(sig_bt[i+1] == T_VOID, "missing Half" );
 458       regs[i].set2(VMRegImpl::stack2reg(dstack));
 459       dstack += 2;
 460       break;
 461     case T_DOUBLE:
 462       assert(sig_bt[i+1] == T_VOID, "missing Half" );
 463       if( freg_arg0 == (uint)i ) {
 464         regs[i].set2(xmm0->as_VMReg());
 465       } else if( freg_arg1 == (uint)i ) {
 466         regs[i].set2(xmm1->as_VMReg());
 467       } else {
 468         regs[i].set2(VMRegImpl::stack2reg(dstack));
 469         dstack += 2;
 470       }
 471       break;
 472     case T_VOID: regs[i].set_bad(); break;
 473       break;
 474     default:
 475       ShouldNotReachHere();
 476       break;
 477     }
 478   }
 479 
 480   // return value can be odd number of VMRegImpl stack slots make multiple of 2
 481   return round_to(stack, 2);
 482 }
 483 
 484 // Patch the callers callsite with entry to compiled code if it exists.
 485 static void patch_callers_callsite(MacroAssembler *masm) {
 486   Label L;
 487   __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);
 488   __ jcc(Assembler::equal, L);
 489   // Schedule the branch target address early.
 490   // Call into the VM to patch the caller, then jump to compiled callee
 491   // rax, isn't live so capture return address while we easily can
 492   __ movptr(rax, Address(rsp, 0));
 493   __ pusha();
 494   __ pushf();
 495 
 496   if (UseSSE == 1) {
 497     __ subptr(rsp, 2*wordSize);
 498     __ movflt(Address(rsp, 0), xmm0);
 499     __ movflt(Address(rsp, wordSize), xmm1);
 500   }
 501   if (UseSSE >= 2) {
 502     __ subptr(rsp, 4*wordSize);
 503     __ movdbl(Address(rsp, 0), xmm0);
 504     __ movdbl(Address(rsp, 2*wordSize), xmm1);
 505   }
 506 #ifdef COMPILER2
 507   // C2 may leave the stack dirty if not in SSE2+ mode
 508   if (UseSSE >= 2) {
 509     __ verify_FPU(0, "c2i transition should have clean FPU stack");
 510   } else {
 511     __ empty_FPU_stack();
 512   }
 513 #endif /* COMPILER2 */
 514 
 515   // VM needs caller's callsite
 516   __ push(rax);
 517   // VM needs target method
 518   __ push(rbx);
 519   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 520   __ addptr(rsp, 2*wordSize);
 521 
 522   if (UseSSE == 1) {
 523     __ movflt(xmm0, Address(rsp, 0));
 524     __ movflt(xmm1, Address(rsp, wordSize));
 525     __ addptr(rsp, 2*wordSize);
 526   }
 527   if (UseSSE >= 2) {
 528     __ movdbl(xmm0, Address(rsp, 0));
 529     __ movdbl(xmm1, Address(rsp, 2*wordSize));
 530     __ addptr(rsp, 4*wordSize);
 531   }
 532 
 533   __ popf();
 534   __ popa();
 535   __ bind(L);
 536 }
 537 
 538 
 539 static void move_c2i_double(MacroAssembler *masm, XMMRegister r, int st_off) {
 540   int next_off = st_off - Interpreter::stackElementSize;
 541   __ movdbl(Address(rsp, next_off), r);
 542 }
 543 
 544 static void gen_c2i_adapter(MacroAssembler *masm,
 545                             int total_args_passed,
 546                             int comp_args_on_stack,
 547                             const BasicType *sig_bt,
 548                             const VMRegPair *regs,
 549                             Label& skip_fixup) {
 550   // Before we get into the guts of the C2I adapter, see if we should be here
 551   // at all.  We've come from compiled code and are attempting to jump to the
 552   // interpreter, which means the caller made a static call to get here
 553   // (vcalls always get a compiled target if there is one).  Check for a
 554   // compiled target.  If there is one, we need to patch the caller's call.
 555   patch_callers_callsite(masm);
 556 
 557   __ bind(skip_fixup);
 558 
 559 #ifdef COMPILER2
 560   // C2 may leave the stack dirty if not in SSE2+ mode
 561   if (UseSSE >= 2) {
 562     __ verify_FPU(0, "c2i transition should have clean FPU stack");
 563   } else {
 564     __ empty_FPU_stack();
 565   }
 566 #endif /* COMPILER2 */
 567 
 568   // Since all args are passed on the stack, total_args_passed * interpreter_
 569   // stack_element_size  is the
 570   // space we need.
 571   int extraspace = total_args_passed * Interpreter::stackElementSize;
 572 
 573   // Get return address
 574   __ pop(rax);
 575 
 576   // set senderSP value
 577   __ movptr(rsi, rsp);
 578 
 579   __ subptr(rsp, extraspace);
 580 
 581   // Now write the args into the outgoing interpreter space
 582   for (int i = 0; i < total_args_passed; i++) {
 583     if (sig_bt[i] == T_VOID) {
 584       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 585       continue;
 586     }
 587 
 588     // st_off points to lowest address on stack.
 589     int st_off = ((total_args_passed - 1) - i) * Interpreter::stackElementSize;
 590     int next_off = st_off - Interpreter::stackElementSize;
 591 
 592     // Say 4 args:
 593     // i   st_off
 594     // 0   12 T_LONG
 595     // 1    8 T_VOID
 596     // 2    4 T_OBJECT
 597     // 3    0 T_BOOL
 598     VMReg r_1 = regs[i].first();
 599     VMReg r_2 = regs[i].second();
 600     if (!r_1->is_valid()) {
 601       assert(!r_2->is_valid(), "");
 602       continue;
 603     }
 604 
 605     if (r_1->is_stack()) {
 606       // memory to memory use fpu stack top
 607       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 608 
 609       if (!r_2->is_valid()) {
 610         __ movl(rdi, Address(rsp, ld_off));
 611         __ movptr(Address(rsp, st_off), rdi);
 612       } else {
 613 
 614         // ld_off == LSW, ld_off+VMRegImpl::stack_slot_size == MSW
 615         // st_off == MSW, st_off-wordSize == LSW
 616 
 617         __ movptr(rdi, Address(rsp, ld_off));
 618         __ movptr(Address(rsp, next_off), rdi);
 619 #ifndef _LP64
 620         __ movptr(rdi, Address(rsp, ld_off + wordSize));
 621         __ movptr(Address(rsp, st_off), rdi);
 622 #else
 623 #ifdef ASSERT
 624         // Overwrite the unused slot with known junk
 625         __ mov64(rax, CONST64(0xdeadffffdeadaaaa));
 626         __ movptr(Address(rsp, st_off), rax);
 627 #endif /* ASSERT */
 628 #endif // _LP64
 629       }
 630     } else if (r_1->is_Register()) {
 631       Register r = r_1->as_Register();
 632       if (!r_2->is_valid()) {
 633         __ movl(Address(rsp, st_off), r);
 634       } else {
 635         // long/double in gpr
 636         NOT_LP64(ShouldNotReachHere());
 637         // Two VMRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 638         // T_DOUBLE and T_LONG use two slots in the interpreter
 639         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 640           // long/double in gpr
 641 #ifdef ASSERT
 642           // Overwrite the unused slot with known junk
 643           LP64_ONLY(__ mov64(rax, CONST64(0xdeadffffdeadaaab)));
 644           __ movptr(Address(rsp, st_off), rax);
 645 #endif /* ASSERT */
 646           __ movptr(Address(rsp, next_off), r);
 647         } else {
 648           __ movptr(Address(rsp, st_off), r);
 649         }
 650       }
 651     } else {
 652       assert(r_1->is_XMMRegister(), "");
 653       if (!r_2->is_valid()) {
 654         __ movflt(Address(rsp, st_off), r_1->as_XMMRegister());
 655       } else {
 656         assert(sig_bt[i] == T_DOUBLE || sig_bt[i] == T_LONG, "wrong type");
 657         move_c2i_double(masm, r_1->as_XMMRegister(), st_off);
 658       }
 659     }
 660   }
 661 
 662   // Schedule the branch target address early.
 663   __ movptr(rcx, Address(rbx, in_bytes(Method::interpreter_entry_offset())));
 664   // And repush original return address
 665   __ push(rax);
 666   __ jmp(rcx);
 667 }
 668 
 669 
 670 static void move_i2c_double(MacroAssembler *masm, XMMRegister r, Register saved_sp, int ld_off) {
 671   int next_val_off = ld_off - Interpreter::stackElementSize;
 672   __ movdbl(r, Address(saved_sp, next_val_off));
 673 }
 674 
 675 static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg,
 676                         address code_start, address code_end,
 677                         Label& L_ok) {
 678   Label L_fail;
 679   __ lea(temp_reg, ExternalAddress(code_start));
 680   __ cmpptr(pc_reg, temp_reg);
 681   __ jcc(Assembler::belowEqual, L_fail);
 682   __ lea(temp_reg, ExternalAddress(code_end));
 683   __ cmpptr(pc_reg, temp_reg);
 684   __ jcc(Assembler::below, L_ok);
 685   __ bind(L_fail);
 686 }
 687 
 688 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 689                                     int total_args_passed,
 690                                     int comp_args_on_stack,
 691                                     const BasicType *sig_bt,
 692                                     const VMRegPair *regs) {
 693   // Note: rsi contains the senderSP on entry. We must preserve it since
 694   // we may do a i2c -> c2i transition if we lose a race where compiled
 695   // code goes non-entrant while we get args ready.
 696 
 697   // Adapters can be frameless because they do not require the caller
 698   // to perform additional cleanup work, such as correcting the stack pointer.
 699   // An i2c adapter is frameless because the *caller* frame, which is interpreted,
 700   // routinely repairs its own stack pointer (from interpreter_frame_last_sp),
 701   // even if a callee has modified the stack pointer.
 702   // A c2i adapter is frameless because the *callee* frame, which is interpreted,
 703   // routinely repairs its caller's stack pointer (from sender_sp, which is set
 704   // up via the senderSP register).
 705   // In other words, if *either* the caller or callee is interpreted, we can
 706   // get the stack pointer repaired after a call.
 707   // This is why c2i and i2c adapters cannot be indefinitely composed.
 708   // In particular, if a c2i adapter were to somehow call an i2c adapter,
 709   // both caller and callee would be compiled methods, and neither would
 710   // clean up the stack pointer changes performed by the two adapters.
 711   // If this happens, control eventually transfers back to the compiled
 712   // caller, but with an uncorrected stack, causing delayed havoc.
 713 
 714   // Pick up the return address
 715   __ movptr(rax, Address(rsp, 0));
 716 
 717   if (VerifyAdapterCalls &&
 718       (Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {
 719     // So, let's test for cascading c2i/i2c adapters right now.
 720     //  assert(Interpreter::contains($return_addr) ||
 721     //         StubRoutines::contains($return_addr),
 722     //         "i2c adapter must return to an interpreter frame");
 723     __ block_comment("verify_i2c { ");
 724     Label L_ok;
 725     if (Interpreter::code() != NULL)
 726       range_check(masm, rax, rdi,
 727                   Interpreter::code()->code_start(), Interpreter::code()->code_end(),
 728                   L_ok);
 729     if (StubRoutines::code1() != NULL)
 730       range_check(masm, rax, rdi,
 731                   StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
 732                   L_ok);
 733     if (StubRoutines::code2() != NULL)
 734       range_check(masm, rax, rdi,
 735                   StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
 736                   L_ok);
 737     const char* msg = "i2c adapter must return to an interpreter frame";
 738     __ block_comment(msg);
 739     __ stop(msg);
 740     __ bind(L_ok);
 741     __ block_comment("} verify_i2ce ");
 742   }
 743 
 744   // Must preserve original SP for loading incoming arguments because
 745   // we need to align the outgoing SP for compiled code.
 746   __ movptr(rdi, rsp);
 747 
 748   // Cut-out for having no stack args.  Since up to 2 int/oop args are passed
 749   // in registers, we will occasionally have no stack args.
 750   int comp_words_on_stack = 0;
 751   if (comp_args_on_stack) {
 752     // Sig words on the stack are greater-than VMRegImpl::stack0.  Those in
 753     // registers are below.  By subtracting stack0, we either get a negative
 754     // number (all values in registers) or the maximum stack slot accessed.
 755     // int comp_args_on_stack = VMRegImpl::reg2stack(max_arg);
 756     // Convert 4-byte stack slots to words.
 757     comp_words_on_stack = round_to(comp_args_on_stack*4, wordSize)>>LogBytesPerWord;
 758     // Round up to miminum stack alignment, in wordSize
 759     comp_words_on_stack = round_to(comp_words_on_stack, 2);
 760     __ subptr(rsp, comp_words_on_stack * wordSize);
 761   }
 762 
 763   // Align the outgoing SP
 764   __ andptr(rsp, -(StackAlignmentInBytes));
 765 
 766   // push the return address on the stack (note that pushing, rather
 767   // than storing it, yields the correct frame alignment for the callee)
 768   __ push(rax);
 769 
 770   // Put saved SP in another register
 771   const Register saved_sp = rax;
 772   __ movptr(saved_sp, rdi);
 773 
 774 
 775   // Will jump to the compiled code just as if compiled code was doing it.
 776   // Pre-load the register-jump target early, to schedule it better.
 777   __ movptr(rdi, Address(rbx, in_bytes(Method::from_compiled_offset())));
 778 
 779   // Now generate the shuffle code.  Pick up all register args and move the
 780   // rest through the floating point stack top.
 781   for (int i = 0; i < total_args_passed; i++) {
 782     if (sig_bt[i] == T_VOID) {
 783       // Longs and doubles are passed in native word order, but misaligned
 784       // in the 32-bit build.
 785       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 786       continue;
 787     }
 788 
 789     // Pick up 0, 1 or 2 words from SP+offset.
 790 
 791     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
 792             "scrambled load targets?");
 793     // Load in argument order going down.
 794     int ld_off = (total_args_passed - i) * Interpreter::stackElementSize;
 795     // Point to interpreter value (vs. tag)
 796     int next_off = ld_off - Interpreter::stackElementSize;
 797     //
 798     //
 799     //
 800     VMReg r_1 = regs[i].first();
 801     VMReg r_2 = regs[i].second();
 802     if (!r_1->is_valid()) {
 803       assert(!r_2->is_valid(), "");
 804       continue;
 805     }
 806     if (r_1->is_stack()) {
 807       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 808       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size + wordSize;
 809 
 810       // We can use rsi as a temp here because compiled code doesn't need rsi as an input
 811       // and if we end up going thru a c2i because of a miss a reasonable value of rsi
 812       // we be generated.
 813       if (!r_2->is_valid()) {
 814         // __ fld_s(Address(saved_sp, ld_off));
 815         // __ fstp_s(Address(rsp, st_off));
 816         __ movl(rsi, Address(saved_sp, ld_off));
 817         __ movptr(Address(rsp, st_off), rsi);
 818       } else {
 819         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 820         // are accessed as negative so LSW is at LOW address
 821 
 822         // ld_off is MSW so get LSW
 823         // st_off is LSW (i.e. reg.first())
 824         // __ fld_d(Address(saved_sp, next_off));
 825         // __ fstp_d(Address(rsp, st_off));
 826         //
 827         // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
 828         // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
 829         // So we must adjust where to pick up the data to match the interpreter.
 830         //
 831         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 832         // are accessed as negative so LSW is at LOW address
 833 
 834         // ld_off is MSW so get LSW
 835         const int offset = (NOT_LP64(true ||) sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 836                            next_off : ld_off;
 837         __ movptr(rsi, Address(saved_sp, offset));
 838         __ movptr(Address(rsp, st_off), rsi);
 839 #ifndef _LP64
 840         __ movptr(rsi, Address(saved_sp, ld_off));
 841         __ movptr(Address(rsp, st_off + wordSize), rsi);
 842 #endif // _LP64
 843       }
 844     } else if (r_1->is_Register()) {  // Register argument
 845       Register r = r_1->as_Register();
 846       assert(r != rax, "must be different");
 847       if (r_2->is_valid()) {
 848         //
 849         // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
 850         // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
 851         // So we must adjust where to pick up the data to match the interpreter.
 852 
 853         const int offset = (NOT_LP64(true ||) sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 854                            next_off : ld_off;
 855 
 856         // this can be a misaligned move
 857         __ movptr(r, Address(saved_sp, offset));
 858 #ifndef _LP64
 859         assert(r_2->as_Register() != rax, "need another temporary register");
 860         // Remember r_1 is low address (and LSB on x86)
 861         // So r_2 gets loaded from high address regardless of the platform
 862         __ movptr(r_2->as_Register(), Address(saved_sp, ld_off));
 863 #endif // _LP64
 864       } else {
 865         __ movl(r, Address(saved_sp, ld_off));
 866       }
 867     } else {
 868       assert(r_1->is_XMMRegister(), "");
 869       if (!r_2->is_valid()) {
 870         __ movflt(r_1->as_XMMRegister(), Address(saved_sp, ld_off));
 871       } else {
 872         move_i2c_double(masm, r_1->as_XMMRegister(), saved_sp, ld_off);
 873       }
 874     }
 875   }
 876 
 877   // 6243940 We might end up in handle_wrong_method if
 878   // the callee is deoptimized as we race thru here. If that
 879   // happens we don't want to take a safepoint because the
 880   // caller frame will look interpreted and arguments are now
 881   // "compiled" so it is much better to make this transition
 882   // invisible to the stack walking code. Unfortunately if
 883   // we try and find the callee by normal means a safepoint
 884   // is possible. So we stash the desired callee in the thread
 885   // and the vm will find there should this case occur.
 886 
 887   __ get_thread(rax);
 888   __ movptr(Address(rax, JavaThread::callee_target_offset()), rbx);
 889 
 890   // move Method* to rax, in case we end up in an c2i adapter.
 891   // the c2i adapters expect Method* in rax, (c2) because c2's
 892   // resolve stubs return the result (the method) in rax,.
 893   // I'd love to fix this.
 894   __ mov(rax, rbx);
 895 
 896   __ jmp(rdi);
 897 }
 898 
 899 // ---------------------------------------------------------------
 900 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 901                                                             int total_args_passed,
 902                                                             int comp_args_on_stack,
 903                                                             const BasicType *sig_bt,
 904                                                             const VMRegPair *regs,
 905                                                             AdapterFingerPrint* fingerprint) {
 906   address i2c_entry = __ pc();
 907 
 908   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 909 
 910   // -------------------------------------------------------------------------
 911   // Generate a C2I adapter.  On entry we know rbx, holds the Method* during calls
 912   // to the interpreter.  The args start out packed in the compiled layout.  They
 913   // need to be unpacked into the interpreter layout.  This will almost always
 914   // require some stack space.  We grow the current (compiled) stack, then repack
 915   // the args.  We  finally end in a jump to the generic interpreter entry point.
 916   // On exit from the interpreter, the interpreter will restore our SP (lest the
 917   // compiled code, which relys solely on SP and not EBP, get sick).
 918 
 919   address c2i_unverified_entry = __ pc();
 920   Label skip_fixup;
 921 
 922   Register holder = rax;
 923   Register receiver = rcx;
 924   Register temp = rbx;
 925 
 926   {
 927 
 928     Label missed;
 929     __ movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 930     __ cmpptr(temp, Address(holder, CompiledICHolder::holder_klass_offset()));
 931     __ movptr(rbx, Address(holder, CompiledICHolder::holder_method_offset()));
 932     __ jcc(Assembler::notEqual, missed);
 933     // Method might have been compiled since the call site was patched to
 934     // interpreted if that is the case treat it as a miss so we can get
 935     // the call site corrected.
 936     __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);
 937     __ jcc(Assembler::equal, skip_fixup);
 938 
 939     __ bind(missed);
 940     __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 941   }
 942 
 943   address c2i_entry = __ pc();
 944 
 945   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
 946 
 947   __ flush();
 948   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry);
 949 }
 950 
 951 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
 952                                          VMRegPair *regs,
 953                                          VMRegPair *regs2,
 954                                          int total_args_passed) {
 955   assert(regs2 == NULL, "not needed on x86");
 956 // We return the amount of VMRegImpl stack slots we need to reserve for all
 957 // the arguments NOT counting out_preserve_stack_slots.
 958 
 959   uint    stack = 0;        // All arguments on stack
 960 
 961   for( int i = 0; i < total_args_passed; i++) {
 962     // From the type and the argument number (count) compute the location
 963     switch( sig_bt[i] ) {
 964     case T_BOOLEAN:
 965     case T_CHAR:
 966     case T_FLOAT:
 967     case T_BYTE:
 968     case T_SHORT:
 969     case T_INT:
 970     case T_OBJECT:
 971     case T_ARRAY:
 972     case T_ADDRESS:
 973     case T_METADATA:
 974       regs[i].set1(VMRegImpl::stack2reg(stack++));
 975       break;
 976     case T_LONG:
 977     case T_DOUBLE: // The stack numbering is reversed from Java
 978       // Since C arguments do not get reversed, the ordering for
 979       // doubles on the stack must be opposite the Java convention
 980       assert(sig_bt[i+1] == T_VOID, "missing Half" );
 981       regs[i].set2(VMRegImpl::stack2reg(stack));
 982       stack += 2;
 983       break;
 984     case T_VOID: regs[i].set_bad(); break;
 985     default:
 986       ShouldNotReachHere();
 987       break;
 988     }
 989   }
 990   return stack;
 991 }
 992 
 993 // A simple move of integer like type
 994 static void simple_move32(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
 995   if (src.first()->is_stack()) {
 996     if (dst.first()->is_stack()) {
 997       // stack to stack
 998       // __ ld(FP, reg2offset(src.first()) + STACK_BIAS, L5);
 999       // __ st(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
1000       __ movl2ptr(rax, Address(rbp, reg2offset_in(src.first())));
1001       __ movptr(Address(rsp, reg2offset_out(dst.first())), rax);
1002     } else {
1003       // stack to reg
1004       __ movl2ptr(dst.first()->as_Register(),  Address(rbp, reg2offset_in(src.first())));
1005     }
1006   } else if (dst.first()->is_stack()) {
1007     // reg to stack
1008     // no need to sign extend on 64bit
1009     __ movptr(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1010   } else {
1011     if (dst.first() != src.first()) {
1012       __ mov(dst.first()->as_Register(), src.first()->as_Register());
1013     }
1014   }
1015 }
1016 
1017 // An oop arg. Must pass a handle not the oop itself
1018 static void object_move(MacroAssembler* masm,
1019                         OopMap* map,
1020                         int oop_handle_offset,
1021                         int framesize_in_slots,
1022                         VMRegPair src,
1023                         VMRegPair dst,
1024                         bool is_receiver,
1025                         int* receiver_offset) {
1026 
1027   // Because of the calling conventions we know that src can be a
1028   // register or a stack location. dst can only be a stack location.
1029 
1030   assert(dst.first()->is_stack(), "must be stack");
1031   // must pass a handle. First figure out the location we use as a handle
1032 
1033   if (src.first()->is_stack()) {
1034     // Oop is already on the stack as an argument
1035     Register rHandle = rax;
1036     Label nil;
1037     __ xorptr(rHandle, rHandle);
1038     __ cmpptr(Address(rbp, reg2offset_in(src.first())), (int32_t)NULL_WORD);
1039     __ jcc(Assembler::equal, nil);
1040     __ lea(rHandle, Address(rbp, reg2offset_in(src.first())));
1041     __ bind(nil);
1042     __ movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
1043 
1044     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1045     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
1046     if (is_receiver) {
1047       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
1048     }
1049   } else {
1050     // Oop is in an a register we must store it to the space we reserve
1051     // on the stack for oop_handles
1052     const Register rOop = src.first()->as_Register();
1053     const Register rHandle = rax;
1054     int oop_slot = (rOop == rcx ? 0 : 1) * VMRegImpl::slots_per_word + oop_handle_offset;
1055     int offset = oop_slot*VMRegImpl::stack_slot_size;
1056     Label skip;
1057     __ movptr(Address(rsp, offset), rOop);
1058     map->set_oop(VMRegImpl::stack2reg(oop_slot));
1059     __ xorptr(rHandle, rHandle);
1060     __ cmpptr(rOop, (int32_t)NULL_WORD);
1061     __ jcc(Assembler::equal, skip);
1062     __ lea(rHandle, Address(rsp, offset));
1063     __ bind(skip);
1064     // Store the handle parameter
1065     __ movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
1066     if (is_receiver) {
1067       *receiver_offset = offset;
1068     }
1069   }
1070 }
1071 
1072 // A float arg may have to do float reg int reg conversion
1073 static void float_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1074   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
1075 
1076   // Because of the calling convention we know that src is either a stack location
1077   // or an xmm register. dst can only be a stack location.
1078 
1079   assert(dst.first()->is_stack() && ( src.first()->is_stack() || src.first()->is_XMMRegister()), "bad parameters");
1080 
1081   if (src.first()->is_stack()) {
1082     __ movl(rax, Address(rbp, reg2offset_in(src.first())));
1083     __ movptr(Address(rsp, reg2offset_out(dst.first())), rax);
1084   } else {
1085     // reg to stack
1086     __ movflt(Address(rsp, reg2offset_out(dst.first())), src.first()->as_XMMRegister());
1087   }
1088 }
1089 
1090 // A long move
1091 static void long_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1092 
1093   // The only legal possibility for a long_move VMRegPair is:
1094   // 1: two stack slots (possibly unaligned)
1095   // as neither the java  or C calling convention will use registers
1096   // for longs.
1097 
1098   if (src.first()->is_stack() && dst.first()->is_stack()) {
1099     assert(src.second()->is_stack() && dst.second()->is_stack(), "must be all stack");
1100     __ movptr(rax, Address(rbp, reg2offset_in(src.first())));
1101     NOT_LP64(__ movptr(rbx, Address(rbp, reg2offset_in(src.second()))));
1102     __ movptr(Address(rsp, reg2offset_out(dst.first())), rax);
1103     NOT_LP64(__ movptr(Address(rsp, reg2offset_out(dst.second())), rbx));
1104   } else {
1105     ShouldNotReachHere();
1106   }
1107 }
1108 
1109 // A double move
1110 static void double_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1111 
1112   // The only legal possibilities for a double_move VMRegPair are:
1113   // The painful thing here is that like long_move a VMRegPair might be
1114 
1115   // Because of the calling convention we know that src is either
1116   //   1: a single physical register (xmm registers only)
1117   //   2: two stack slots (possibly unaligned)
1118   // dst can only be a pair of stack slots.
1119 
1120   assert(dst.first()->is_stack() && (src.first()->is_XMMRegister() || src.first()->is_stack()), "bad args");
1121 
1122   if (src.first()->is_stack()) {
1123     // source is all stack
1124     __ movptr(rax, Address(rbp, reg2offset_in(src.first())));
1125     NOT_LP64(__ movptr(rbx, Address(rbp, reg2offset_in(src.second()))));
1126     __ movptr(Address(rsp, reg2offset_out(dst.first())), rax);
1127     NOT_LP64(__ movptr(Address(rsp, reg2offset_out(dst.second())), rbx));
1128   } else {
1129     // reg to stack
1130     // No worries about stack alignment
1131     __ movdbl(Address(rsp, reg2offset_out(dst.first())), src.first()->as_XMMRegister());
1132   }
1133 }
1134 
1135 
1136 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1137   // We always ignore the frame_slots arg and just use the space just below frame pointer
1138   // which by this time is free to use
1139   switch (ret_type) {
1140   case T_FLOAT:
1141     __ fstp_s(Address(rbp, -wordSize));
1142     break;
1143   case T_DOUBLE:
1144     __ fstp_d(Address(rbp, -2*wordSize));
1145     break;
1146   case T_VOID:  break;
1147   case T_LONG:
1148     __ movptr(Address(rbp, -wordSize), rax);
1149     NOT_LP64(__ movptr(Address(rbp, -2*wordSize), rdx));
1150     break;
1151   default: {
1152     __ movptr(Address(rbp, -wordSize), rax);
1153     }
1154   }
1155 }
1156 
1157 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1158   // We always ignore the frame_slots arg and just use the space just below frame pointer
1159   // which by this time is free to use
1160   switch (ret_type) {
1161   case T_FLOAT:
1162     __ fld_s(Address(rbp, -wordSize));
1163     break;
1164   case T_DOUBLE:
1165     __ fld_d(Address(rbp, -2*wordSize));
1166     break;
1167   case T_LONG:
1168     __ movptr(rax, Address(rbp, -wordSize));
1169     NOT_LP64(__ movptr(rdx, Address(rbp, -2*wordSize)));
1170     break;
1171   case T_VOID:  break;
1172   default: {
1173     __ movptr(rax, Address(rbp, -wordSize));
1174     }
1175   }
1176 }
1177 
1178 
1179 static void save_or_restore_arguments(MacroAssembler* masm,
1180                                       const int stack_slots,
1181                                       const int total_in_args,
1182                                       const int arg_save_area,
1183                                       OopMap* map,
1184                                       VMRegPair* in_regs,
1185                                       BasicType* in_sig_bt) {
1186   // if map is non-NULL then the code should store the values,
1187   // otherwise it should load them.
1188   int handle_index = 0;
1189   // Save down double word first
1190   for ( int i = 0; i < total_in_args; i++) {
1191     if (in_regs[i].first()->is_XMMRegister() && in_sig_bt[i] == T_DOUBLE) {
1192       int slot = handle_index * VMRegImpl::slots_per_word + arg_save_area;
1193       int offset = slot * VMRegImpl::stack_slot_size;
1194       handle_index += 2;
1195       assert(handle_index <= stack_slots, "overflow");
1196       if (map != NULL) {
1197         __ movdbl(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1198       } else {
1199         __ movdbl(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1200       }
1201     }
1202     if (in_regs[i].first()->is_Register() && in_sig_bt[i] == T_LONG) {
1203       int slot = handle_index * VMRegImpl::slots_per_word + arg_save_area;
1204       int offset = slot * VMRegImpl::stack_slot_size;
1205       handle_index += 2;
1206       assert(handle_index <= stack_slots, "overflow");
1207       if (map != NULL) {
1208         __ movl(Address(rsp, offset), in_regs[i].first()->as_Register());
1209         if (in_regs[i].second()->is_Register()) {
1210           __ movl(Address(rsp, offset + 4), in_regs[i].second()->as_Register());
1211         }
1212       } else {
1213         __ movl(in_regs[i].first()->as_Register(), Address(rsp, offset));
1214         if (in_regs[i].second()->is_Register()) {
1215           __ movl(in_regs[i].second()->as_Register(), Address(rsp, offset + 4));
1216         }
1217       }
1218     }
1219   }
1220   // Save or restore single word registers
1221   for ( int i = 0; i < total_in_args; i++) {
1222     if (in_regs[i].first()->is_Register()) {
1223       int slot = handle_index++ * VMRegImpl::slots_per_word + arg_save_area;
1224       int offset = slot * VMRegImpl::stack_slot_size;
1225       assert(handle_index <= stack_slots, "overflow");
1226       if (in_sig_bt[i] == T_ARRAY && map != NULL) {
1227         map->set_oop(VMRegImpl::stack2reg(slot));;
1228       }
1229 
1230       // Value is in an input register pass we must flush it to the stack
1231       const Register reg = in_regs[i].first()->as_Register();
1232       switch (in_sig_bt[i]) {
1233         case T_ARRAY:
1234           if (map != NULL) {
1235             __ movptr(Address(rsp, offset), reg);
1236           } else {
1237             __ movptr(reg, Address(rsp, offset));
1238           }
1239           break;
1240         case T_BOOLEAN:
1241         case T_CHAR:
1242         case T_BYTE:
1243         case T_SHORT:
1244         case T_INT:
1245           if (map != NULL) {
1246             __ movl(Address(rsp, offset), reg);
1247           } else {
1248             __ movl(reg, Address(rsp, offset));
1249           }
1250           break;
1251         case T_OBJECT:
1252         default: ShouldNotReachHere();
1253       }
1254     } else if (in_regs[i].first()->is_XMMRegister()) {
1255       if (in_sig_bt[i] == T_FLOAT) {
1256         int slot = handle_index++ * VMRegImpl::slots_per_word + arg_save_area;
1257         int offset = slot * VMRegImpl::stack_slot_size;
1258         assert(handle_index <= stack_slots, "overflow");
1259         if (map != NULL) {
1260           __ movflt(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1261         } else {
1262           __ movflt(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1263         }
1264       }
1265     } else if (in_regs[i].first()->is_stack()) {
1266       if (in_sig_bt[i] == T_ARRAY && map != NULL) {
1267         int offset_in_older_frame = in_regs[i].first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1268         map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
1269       }
1270     }
1271   }
1272 }
1273 
1274 // Check GCLocker::needs_gc and enter the runtime if it's true.  This
1275 // keeps a new JNI critical region from starting until a GC has been
1276 // forced.  Save down any oops in registers and describe them in an
1277 // OopMap.
1278 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1279                                                Register thread,
1280                                                int stack_slots,
1281                                                int total_c_args,
1282                                                int total_in_args,
1283                                                int arg_save_area,
1284                                                OopMapSet* oop_maps,
1285                                                VMRegPair* in_regs,
1286                                                BasicType* in_sig_bt) {
1287   __ block_comment("check GCLocker::needs_gc");
1288   Label cont;
1289   __ cmp8(ExternalAddress((address)GCLocker::needs_gc_address()), false);
1290   __ jcc(Assembler::equal, cont);
1291 
1292   // Save down any incoming oops and call into the runtime to halt for a GC
1293 
1294   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1295 
1296   save_or_restore_arguments(masm, stack_slots, total_in_args,
1297                             arg_save_area, map, in_regs, in_sig_bt);
1298 
1299   address the_pc = __ pc();
1300   oop_maps->add_gc_map( __ offset(), map);
1301   __ set_last_Java_frame(thread, rsp, noreg, the_pc);
1302 
1303   __ block_comment("block_for_jni_critical");
1304   __ push(thread);
1305   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::block_for_jni_critical)));
1306   __ increment(rsp, wordSize);
1307 
1308   __ get_thread(thread);
1309   __ reset_last_Java_frame(thread, false, true);
1310 
1311   save_or_restore_arguments(masm, stack_slots, total_in_args,
1312                             arg_save_area, NULL, in_regs, in_sig_bt);
1313 
1314   __ bind(cont);
1315 #ifdef ASSERT
1316   if (StressCriticalJNINatives) {
1317     // Stress register saving
1318     OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1319     save_or_restore_arguments(masm, stack_slots, total_in_args,
1320                               arg_save_area, map, in_regs, in_sig_bt);
1321     // Destroy argument registers
1322     for (int i = 0; i < total_in_args - 1; i++) {
1323       if (in_regs[i].first()->is_Register()) {
1324         const Register reg = in_regs[i].first()->as_Register();
1325         __ xorptr(reg, reg);
1326       } else if (in_regs[i].first()->is_XMMRegister()) {
1327         __ xorpd(in_regs[i].first()->as_XMMRegister(), in_regs[i].first()->as_XMMRegister());
1328       } else if (in_regs[i].first()->is_FloatRegister()) {
1329         ShouldNotReachHere();
1330       } else if (in_regs[i].first()->is_stack()) {
1331         // Nothing to do
1332       } else {
1333         ShouldNotReachHere();
1334       }
1335       if (in_sig_bt[i] == T_LONG || in_sig_bt[i] == T_DOUBLE) {
1336         i++;
1337       }
1338     }
1339 
1340     save_or_restore_arguments(masm, stack_slots, total_in_args,
1341                               arg_save_area, NULL, in_regs, in_sig_bt);
1342   }
1343 #endif
1344 }
1345 
1346 // Unpack an array argument into a pointer to the body and the length
1347 // if the array is non-null, otherwise pass 0 for both.
1348 static void unpack_array_argument(MacroAssembler* masm, VMRegPair reg, BasicType in_elem_type, VMRegPair body_arg, VMRegPair length_arg) {
1349   Register tmp_reg = rax;
1350   assert(!body_arg.first()->is_Register() || body_arg.first()->as_Register() != tmp_reg,
1351          "possible collision");
1352   assert(!length_arg.first()->is_Register() || length_arg.first()->as_Register() != tmp_reg,
1353          "possible collision");
1354 
1355   // Pass the length, ptr pair
1356   Label is_null, done;
1357   VMRegPair tmp(tmp_reg->as_VMReg());
1358   if (reg.first()->is_stack()) {
1359     // Load the arg up from the stack
1360     simple_move32(masm, reg, tmp);
1361     reg = tmp;
1362   }
1363   __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
1364   __ jccb(Assembler::equal, is_null);
1365   __ lea(tmp_reg, Address(reg.first()->as_Register(), arrayOopDesc::base_offset_in_bytes(in_elem_type)));
1366   simple_move32(masm, tmp, body_arg);
1367   // load the length relative to the body.
1368   __ movl(tmp_reg, Address(tmp_reg, arrayOopDesc::length_offset_in_bytes() -
1369                            arrayOopDesc::base_offset_in_bytes(in_elem_type)));
1370   simple_move32(masm, tmp, length_arg);
1371   __ jmpb(done);
1372   __ bind(is_null);
1373   // Pass zeros
1374   __ xorptr(tmp_reg, tmp_reg);
1375   simple_move32(masm, tmp, body_arg);
1376   simple_move32(masm, tmp, length_arg);
1377   __ bind(done);
1378 }
1379 
1380 static void verify_oop_args(MacroAssembler* masm,
1381                             methodHandle method,
1382                             const BasicType* sig_bt,
1383                             const VMRegPair* regs) {
1384   Register temp_reg = rbx;  // not part of any compiled calling seq
1385   if (VerifyOops) {
1386     for (int i = 0; i < method->size_of_parameters(); i++) {
1387       if (sig_bt[i] == T_OBJECT ||
1388           sig_bt[i] == T_ARRAY) {
1389         VMReg r = regs[i].first();
1390         assert(r->is_valid(), "bad oop arg");
1391         if (r->is_stack()) {
1392           __ movptr(temp_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1393           __ verify_oop(temp_reg);
1394         } else {
1395           __ verify_oop(r->as_Register());
1396         }
1397       }
1398     }
1399   }
1400 }
1401 
1402 static void gen_special_dispatch(MacroAssembler* masm,
1403                                  methodHandle method,
1404                                  const BasicType* sig_bt,
1405                                  const VMRegPair* regs) {
1406   verify_oop_args(masm, method, sig_bt, regs);
1407   vmIntrinsics::ID iid = method->intrinsic_id();
1408 
1409   // Now write the args into the outgoing interpreter space
1410   bool     has_receiver   = false;
1411   Register receiver_reg   = noreg;
1412   int      member_arg_pos = -1;
1413   Register member_reg     = noreg;
1414   int      ref_kind       = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1415   if (ref_kind != 0) {
1416     member_arg_pos = method->size_of_parameters() - 1;  // trailing MemberName argument
1417     member_reg = rbx;  // known to be free at this point
1418     has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1419   } else if (iid == vmIntrinsics::_invokeBasic) {
1420     has_receiver = true;
1421   } else {
1422     fatal("unexpected intrinsic id %d", iid);
1423   }
1424 
1425   if (member_reg != noreg) {
1426     // Load the member_arg into register, if necessary.
1427     SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1428     VMReg r = regs[member_arg_pos].first();
1429     if (r->is_stack()) {
1430       __ movptr(member_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1431     } else {
1432       // no data motion is needed
1433       member_reg = r->as_Register();
1434     }
1435   }
1436 
1437   if (has_receiver) {
1438     // Make sure the receiver is loaded into a register.
1439     assert(method->size_of_parameters() > 0, "oob");
1440     assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1441     VMReg r = regs[0].first();
1442     assert(r->is_valid(), "bad receiver arg");
1443     if (r->is_stack()) {
1444       // Porting note:  This assumes that compiled calling conventions always
1445       // pass the receiver oop in a register.  If this is not true on some
1446       // platform, pick a temp and load the receiver from stack.
1447       fatal("receiver always in a register");
1448       receiver_reg = rcx;  // known to be free at this point
1449       __ movptr(receiver_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1450     } else {
1451       // no data motion is needed
1452       receiver_reg = r->as_Register();
1453     }
1454   }
1455 
1456   // Figure out which address we are really jumping to:
1457   MethodHandles::generate_method_handle_dispatch(masm, iid,
1458                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
1459 }
1460 
1461 // ---------------------------------------------------------------------------
1462 // Generate a native wrapper for a given method.  The method takes arguments
1463 // in the Java compiled code convention, marshals them to the native
1464 // convention (handlizes oops, etc), transitions to native, makes the call,
1465 // returns to java state (possibly blocking), unhandlizes any result and
1466 // returns.
1467 //
1468 // Critical native functions are a shorthand for the use of
1469 // GetPrimtiveArrayCritical and disallow the use of any other JNI
1470 // functions.  The wrapper is expected to unpack the arguments before
1471 // passing them to the callee and perform checks before and after the
1472 // native call to ensure that they GCLocker
1473 // lock_critical/unlock_critical semantics are followed.  Some other
1474 // parts of JNI setup are skipped like the tear down of the JNI handle
1475 // block and the check for pending exceptions it's impossible for them
1476 // to be thrown.
1477 //
1478 // They are roughly structured like this:
1479 //    if (GCLocker::needs_gc())
1480 //      SharedRuntime::block_for_jni_critical();
1481 //    tranistion to thread_in_native
1482 //    unpack arrray arguments and call native entry point
1483 //    check for safepoint in progress
1484 //    check if any thread suspend flags are set
1485 //      call into JVM and possible unlock the JNI critical
1486 //      if a GC was suppressed while in the critical native.
1487 //    transition back to thread_in_Java
1488 //    return to caller
1489 //
1490 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
1491                                                 const methodHandle& method,
1492                                                 int compile_id,
1493                                                 BasicType* in_sig_bt,
1494                                                 VMRegPair* in_regs,
1495                                                 BasicType ret_type) {
1496   if (method->is_method_handle_intrinsic()) {
1497     vmIntrinsics::ID iid = method->intrinsic_id();
1498     intptr_t start = (intptr_t)__ pc();
1499     int vep_offset = ((intptr_t)__ pc()) - start;
1500     gen_special_dispatch(masm,
1501                          method,
1502                          in_sig_bt,
1503                          in_regs);
1504     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
1505     __ flush();
1506     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
1507     return nmethod::new_native_nmethod(method,
1508                                        compile_id,
1509                                        masm->code(),
1510                                        vep_offset,
1511                                        frame_complete,
1512                                        stack_slots / VMRegImpl::slots_per_word,
1513                                        in_ByteSize(-1),
1514                                        in_ByteSize(-1),
1515                                        (OopMapSet*)NULL);
1516   }
1517   bool is_critical_native = true;
1518   address native_func = method->critical_native_function();
1519   if (native_func == NULL) {
1520     native_func = method->native_function();
1521     is_critical_native = false;
1522   }
1523   assert(native_func != NULL, "must have function");
1524 
1525   // An OopMap for lock (and class if static)
1526   OopMapSet *oop_maps = new OopMapSet();
1527 
1528   // We have received a description of where all the java arg are located
1529   // on entry to the wrapper. We need to convert these args to where
1530   // the jni function will expect them. To figure out where they go
1531   // we convert the java signature to a C signature by inserting
1532   // the hidden arguments as arg[0] and possibly arg[1] (static method)
1533 
1534   const int total_in_args = method->size_of_parameters();
1535   int total_c_args = total_in_args;
1536   if (!is_critical_native) {
1537     total_c_args += 1;
1538     if (method->is_static()) {
1539       total_c_args++;
1540     }
1541   } else {
1542     for (int i = 0; i < total_in_args; i++) {
1543       if (in_sig_bt[i] == T_ARRAY) {
1544         total_c_args++;
1545       }
1546     }
1547   }
1548 
1549   BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1550   VMRegPair* out_regs   = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1551   BasicType* in_elem_bt = NULL;
1552 
1553   int argc = 0;
1554   if (!is_critical_native) {
1555     out_sig_bt[argc++] = T_ADDRESS;
1556     if (method->is_static()) {
1557       out_sig_bt[argc++] = T_OBJECT;
1558     }
1559 
1560     for (int i = 0; i < total_in_args ; i++ ) {
1561       out_sig_bt[argc++] = in_sig_bt[i];
1562     }
1563   } else {
1564     Thread* THREAD = Thread::current();
1565     in_elem_bt = NEW_RESOURCE_ARRAY(BasicType, total_in_args);
1566     SignatureStream ss(method->signature());
1567     for (int i = 0; i < total_in_args ; i++ ) {
1568       if (in_sig_bt[i] == T_ARRAY) {
1569         // Arrays are passed as int, elem* pair
1570         out_sig_bt[argc++] = T_INT;
1571         out_sig_bt[argc++] = T_ADDRESS;
1572         Symbol* atype = ss.as_symbol(CHECK_NULL);
1573         const char* at = atype->as_C_string();
1574         if (strlen(at) == 2) {
1575           assert(at[0] == '[', "must be");
1576           switch (at[1]) {
1577             case 'B': in_elem_bt[i]  = T_BYTE; break;
1578             case 'C': in_elem_bt[i]  = T_CHAR; break;
1579             case 'D': in_elem_bt[i]  = T_DOUBLE; break;
1580             case 'F': in_elem_bt[i]  = T_FLOAT; break;
1581             case 'I': in_elem_bt[i]  = T_INT; break;
1582             case 'J': in_elem_bt[i]  = T_LONG; break;
1583             case 'S': in_elem_bt[i]  = T_SHORT; break;
1584             case 'Z': in_elem_bt[i]  = T_BOOLEAN; break;
1585             default: ShouldNotReachHere();
1586           }
1587         }
1588       } else {
1589         out_sig_bt[argc++] = in_sig_bt[i];
1590         in_elem_bt[i] = T_VOID;
1591       }
1592       if (in_sig_bt[i] != T_VOID) {
1593         assert(in_sig_bt[i] == ss.type(), "must match");
1594         ss.next();
1595       }
1596     }
1597   }
1598 
1599   // Now figure out where the args must be stored and how much stack space
1600   // they require.
1601   int out_arg_slots;
1602   out_arg_slots = c_calling_convention(out_sig_bt, out_regs, NULL, total_c_args);
1603 
1604   // Compute framesize for the wrapper.  We need to handlize all oops in
1605   // registers a max of 2 on x86.
1606 
1607   // Calculate the total number of stack slots we will need.
1608 
1609   // First count the abi requirement plus all of the outgoing args
1610   int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
1611 
1612   // Now the space for the inbound oop handle area
1613   int total_save_slots = 2 * VMRegImpl::slots_per_word; // 2 arguments passed in registers
1614   if (is_critical_native) {
1615     // Critical natives may have to call out so they need a save area
1616     // for register arguments.
1617     int double_slots = 0;
1618     int single_slots = 0;
1619     for ( int i = 0; i < total_in_args; i++) {
1620       if (in_regs[i].first()->is_Register()) {
1621         const Register reg = in_regs[i].first()->as_Register();
1622         switch (in_sig_bt[i]) {
1623           case T_ARRAY:  // critical array (uses 2 slots on LP64)
1624           case T_BOOLEAN:
1625           case T_BYTE:
1626           case T_SHORT:
1627           case T_CHAR:
1628           case T_INT:  single_slots++; break;
1629           case T_LONG: double_slots++; break;
1630           default:  ShouldNotReachHere();
1631         }
1632       } else if (in_regs[i].first()->is_XMMRegister()) {
1633         switch (in_sig_bt[i]) {
1634           case T_FLOAT:  single_slots++; break;
1635           case T_DOUBLE: double_slots++; break;
1636           default:  ShouldNotReachHere();
1637         }
1638       } else if (in_regs[i].first()->is_FloatRegister()) {
1639         ShouldNotReachHere();
1640       }
1641     }
1642     total_save_slots = double_slots * 2 + single_slots;
1643     // align the save area
1644     if (double_slots != 0) {
1645       stack_slots = round_to(stack_slots, 2);
1646     }
1647   }
1648 
1649   int oop_handle_offset = stack_slots;
1650   stack_slots += total_save_slots;
1651 
1652   // Now any space we need for handlizing a klass if static method
1653 
1654   int klass_slot_offset = 0;
1655   int klass_offset = -1;
1656   int lock_slot_offset = 0;
1657   bool is_static = false;
1658 
1659   if (method->is_static()) {
1660     klass_slot_offset = stack_slots;
1661     stack_slots += VMRegImpl::slots_per_word;
1662     klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size;
1663     is_static = true;
1664   }
1665 
1666   // Plus a lock if needed
1667 
1668   if (method->is_synchronized()) {
1669     lock_slot_offset = stack_slots;
1670     stack_slots += VMRegImpl::slots_per_word;
1671   }
1672 
1673   // Now a place (+2) to save return values or temp during shuffling
1674   // + 2 for return address (which we own) and saved rbp,
1675   stack_slots += 4;
1676 
1677   // Ok The space we have allocated will look like:
1678   //
1679   //
1680   // FP-> |                     |
1681   //      |---------------------|
1682   //      | 2 slots for moves   |
1683   //      |---------------------|
1684   //      | lock box (if sync)  |
1685   //      |---------------------| <- lock_slot_offset  (-lock_slot_rbp_offset)
1686   //      | klass (if static)   |
1687   //      |---------------------| <- klass_slot_offset
1688   //      | oopHandle area      |
1689   //      |---------------------| <- oop_handle_offset (a max of 2 registers)
1690   //      | outbound memory     |
1691   //      | based arguments     |
1692   //      |                     |
1693   //      |---------------------|
1694   //      |                     |
1695   // SP-> | out_preserved_slots |
1696   //
1697   //
1698   // ****************************************************************************
1699   // WARNING - on Windows Java Natives use pascal calling convention and pop the
1700   // arguments off of the stack after the jni call. Before the call we can use
1701   // instructions that are SP relative. After the jni call we switch to FP
1702   // relative instructions instead of re-adjusting the stack on windows.
1703   // ****************************************************************************
1704 
1705 
1706   // Now compute actual number of stack words we need rounding to make
1707   // stack properly aligned.
1708   stack_slots = round_to(stack_slots, StackAlignmentInSlots);
1709 
1710   int stack_size = stack_slots * VMRegImpl::stack_slot_size;
1711 
1712   intptr_t start = (intptr_t)__ pc();
1713 
1714   // First thing make an ic check to see if we should even be here
1715 
1716   // We are free to use all registers as temps without saving them and
1717   // restoring them except rbp. rbp is the only callee save register
1718   // as far as the interpreter and the compiler(s) are concerned.
1719 
1720 
1721   const Register ic_reg = rax;
1722   const Register receiver = rcx;
1723   Label hit;
1724   Label exception_pending;
1725 
1726   __ verify_oop(receiver);
1727   __ cmpptr(ic_reg, Address(receiver, oopDesc::klass_offset_in_bytes()));
1728   __ jcc(Assembler::equal, hit);
1729 
1730   __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1731 
1732   // verified entry must be aligned for code patching.
1733   // and the first 5 bytes must be in the same cache line
1734   // if we align at 8 then we will be sure 5 bytes are in the same line
1735   __ align(8);
1736 
1737   __ bind(hit);
1738 
1739   int vep_offset = ((intptr_t)__ pc()) - start;
1740 
1741 #ifdef COMPILER1
1742   if (InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) {
1743     // Object.hashCode can pull the hashCode from the header word
1744     // instead of doing a full VM transition once it's been computed.
1745     // Since hashCode is usually polymorphic at call sites we can't do
1746     // this optimization at the call site without a lot of work.
1747     Label slowCase;
1748     Register receiver = rcx;
1749     Register result = rax;
1750     __ movptr(result, Address(receiver, oopDesc::mark_offset_in_bytes()));
1751 
1752     // check if locked
1753     __ testptr(result, markOopDesc::unlocked_value);
1754     __ jcc (Assembler::zero, slowCase);
1755 
1756     if (UseBiasedLocking) {
1757       // Check if biased and fall through to runtime if so
1758       __ testptr(result, markOopDesc::biased_lock_bit_in_place);
1759       __ jcc (Assembler::notZero, slowCase);
1760     }
1761 
1762     // get hash
1763     __ andptr(result, markOopDesc::hash_mask_in_place);
1764     // test if hashCode exists
1765     __ jcc  (Assembler::zero, slowCase);
1766     __ shrptr(result, markOopDesc::hash_shift);
1767     __ ret(0);
1768     __ bind (slowCase);
1769   }
1770 #endif // COMPILER1
1771 
1772   // The instruction at the verified entry point must be 5 bytes or longer
1773   // because it can be patched on the fly by make_non_entrant. The stack bang
1774   // instruction fits that requirement.
1775 
1776   // Generate stack overflow check
1777 
1778   if (UseStackBanging) {
1779     __ bang_stack_with_offset((int)JavaThread::stack_shadow_zone_size());
1780   } else {
1781     // need a 5 byte instruction to allow MT safe patching to non-entrant
1782     __ fat_nop();
1783   }
1784 
1785   // Generate a new frame for the wrapper.
1786   __ enter();
1787   // -2 because return address is already present and so is saved rbp
1788   __ subptr(rsp, stack_size - 2*wordSize);
1789 
1790   // Frame is now completed as far as size and linkage.
1791   int frame_complete = ((intptr_t)__ pc()) - start;
1792 
1793   if (UseRTMLocking) {
1794     // Abort RTM transaction before calling JNI
1795     // because critical section will be large and will be
1796     // aborted anyway. Also nmethod could be deoptimized.
1797     __ xabort(0);
1798   }
1799 
1800   // Calculate the difference between rsp and rbp,. We need to know it
1801   // after the native call because on windows Java Natives will pop
1802   // the arguments and it is painful to do rsp relative addressing
1803   // in a platform independent way. So after the call we switch to
1804   // rbp, relative addressing.
1805 
1806   int fp_adjustment = stack_size - 2*wordSize;
1807 
1808 #ifdef COMPILER2
1809   // C2 may leave the stack dirty if not in SSE2+ mode
1810   if (UseSSE >= 2) {
1811     __ verify_FPU(0, "c2i transition should have clean FPU stack");
1812   } else {
1813     __ empty_FPU_stack();
1814   }
1815 #endif /* COMPILER2 */
1816 
1817   // Compute the rbp, offset for any slots used after the jni call
1818 
1819   int lock_slot_rbp_offset = (lock_slot_offset*VMRegImpl::stack_slot_size) - fp_adjustment;
1820 
1821   // We use rdi as a thread pointer because it is callee save and
1822   // if we load it once it is usable thru the entire wrapper
1823   const Register thread = rdi;
1824 
1825   // We use rsi as the oop handle for the receiver/klass
1826   // It is callee save so it survives the call to native
1827 
1828   const Register oop_handle_reg = rsi;
1829 
1830   __ get_thread(thread);
1831 
1832   if (is_critical_native) {
1833     check_needs_gc_for_critical_native(masm, thread, stack_slots, total_c_args, total_in_args,
1834                                        oop_handle_offset, oop_maps, in_regs, in_sig_bt);
1835   }
1836 
1837   //
1838   // We immediately shuffle the arguments so that any vm call we have to
1839   // make from here on out (sync slow path, jvmti, etc.) we will have
1840   // captured the oops from our caller and have a valid oopMap for
1841   // them.
1842 
1843   // -----------------
1844   // The Grand Shuffle
1845   //
1846   // Natives require 1 or 2 extra arguments over the normal ones: the JNIEnv*
1847   // and, if static, the class mirror instead of a receiver.  This pretty much
1848   // guarantees that register layout will not match (and x86 doesn't use reg
1849   // parms though amd does).  Since the native abi doesn't use register args
1850   // and the java conventions does we don't have to worry about collisions.
1851   // All of our moved are reg->stack or stack->stack.
1852   // We ignore the extra arguments during the shuffle and handle them at the
1853   // last moment. The shuffle is described by the two calling convention
1854   // vectors we have in our possession. We simply walk the java vector to
1855   // get the source locations and the c vector to get the destinations.
1856 
1857   int c_arg = is_critical_native ? 0 : (method->is_static() ? 2 : 1 );
1858 
1859   // Record rsp-based slot for receiver on stack for non-static methods
1860   int receiver_offset = -1;
1861 
1862   // This is a trick. We double the stack slots so we can claim
1863   // the oops in the caller's frame. Since we are sure to have
1864   // more args than the caller doubling is enough to make
1865   // sure we can capture all the incoming oop args from the
1866   // caller.
1867   //
1868   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1869 
1870   // Mark location of rbp,
1871   // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, rbp->as_VMReg());
1872 
1873   // We know that we only have args in at most two integer registers (rcx, rdx). So rax, rbx
1874   // Are free to temporaries if we have to do  stack to steck moves.
1875   // All inbound args are referenced based on rbp, and all outbound args via rsp.
1876 
1877   for (int i = 0; i < total_in_args ; i++, c_arg++ ) {
1878     switch (in_sig_bt[i]) {
1879       case T_ARRAY:
1880         if (is_critical_native) {
1881           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
1882           c_arg++;
1883           break;
1884         }
1885       case T_OBJECT:
1886         assert(!is_critical_native, "no oop arguments");
1887         object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1888                     ((i == 0) && (!is_static)),
1889                     &receiver_offset);
1890         break;
1891       case T_VOID:
1892         break;
1893 
1894       case T_FLOAT:
1895         float_move(masm, in_regs[i], out_regs[c_arg]);
1896           break;
1897 
1898       case T_DOUBLE:
1899         assert( i + 1 < total_in_args &&
1900                 in_sig_bt[i + 1] == T_VOID &&
1901                 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1902         double_move(masm, in_regs[i], out_regs[c_arg]);
1903         break;
1904 
1905       case T_LONG :
1906         long_move(masm, in_regs[i], out_regs[c_arg]);
1907         break;
1908 
1909       case T_ADDRESS: assert(false, "found T_ADDRESS in java args");
1910 
1911       default:
1912         simple_move32(masm, in_regs[i], out_regs[c_arg]);
1913     }
1914   }
1915 
1916   // Pre-load a static method's oop into rsi.  Used both by locking code and
1917   // the normal JNI call code.
1918   if (method->is_static() && !is_critical_native) {
1919 
1920     //  load opp into a register
1921     __ movoop(oop_handle_reg, JNIHandles::make_local(method->method_holder()->java_mirror()));
1922 
1923     // Now handlize the static class mirror it's known not-null.
1924     __ movptr(Address(rsp, klass_offset), oop_handle_reg);
1925     map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
1926 
1927     // Now get the handle
1928     __ lea(oop_handle_reg, Address(rsp, klass_offset));
1929     // store the klass handle as second argument
1930     __ movptr(Address(rsp, wordSize), oop_handle_reg);
1931   }
1932 
1933   // Change state to native (we save the return address in the thread, since it might not
1934   // be pushed on the stack when we do a a stack traversal). It is enough that the pc()
1935   // points into the right code segment. It does not have to be the correct return pc.
1936   // We use the same pc/oopMap repeatedly when we call out
1937 
1938   intptr_t the_pc = (intptr_t) __ pc();
1939   oop_maps->add_gc_map(the_pc - start, map);
1940 
1941   __ set_last_Java_frame(thread, rsp, noreg, (address)the_pc);
1942 
1943 
1944   // We have all of the arguments setup at this point. We must not touch any register
1945   // argument registers at this point (what if we save/restore them there are no oop?
1946 
1947   {
1948     SkipIfEqual skip_if(masm, &DTraceMethodProbes, 0);
1949     __ mov_metadata(rax, method());
1950     __ call_VM_leaf(
1951          CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
1952          thread, rax);
1953   }
1954 
1955   // RedefineClasses() tracing support for obsolete method entry
1956   if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
1957     __ mov_metadata(rax, method());
1958     __ call_VM_leaf(
1959          CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1960          thread, rax);
1961   }
1962 
1963   // These are register definitions we need for locking/unlocking
1964   const Register swap_reg = rax;  // Must use rax, for cmpxchg instruction
1965   const Register obj_reg  = rcx;  // Will contain the oop
1966   const Register lock_reg = rdx;  // Address of compiler lock object (BasicLock)
1967 
1968   Label slow_path_lock;
1969   Label lock_done;
1970 
1971   // Lock a synchronized method
1972   if (method->is_synchronized()) {
1973     assert(!is_critical_native, "unhandled");
1974 
1975 
1976     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1977 
1978     // Get the handle (the 2nd argument)
1979     __ movptr(oop_handle_reg, Address(rsp, wordSize));
1980 
1981     // Get address of the box
1982 
1983     __ lea(lock_reg, Address(rbp, lock_slot_rbp_offset));
1984 
1985     // Load the oop from the handle
1986     __ movptr(obj_reg, Address(oop_handle_reg, 0));
1987 
1988     if (UseBiasedLocking) {
1989       // Note that oop_handle_reg is trashed during this call
1990       __ biased_locking_enter(lock_reg, obj_reg, swap_reg, oop_handle_reg, false, lock_done, &slow_path_lock);
1991     }
1992 
1993     // Load immediate 1 into swap_reg %rax,
1994     __ movptr(swap_reg, 1);
1995 
1996     // Load (object->mark() | 1) into swap_reg %rax,
1997     __ orptr(swap_reg, Address(obj_reg, 0));
1998 
1999     // Save (object->mark() | 1) into BasicLock's displaced header
2000     __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2001 
2002     if (os::is_MP()) {
2003       __ lock();
2004     }
2005 
2006     // src -> dest iff dest == rax, else rax, <- dest
2007     // *obj_reg = lock_reg iff *obj_reg == rax, else rax, = *(obj_reg)
2008     __ cmpxchgptr(lock_reg, Address(obj_reg, 0));
2009     __ jcc(Assembler::equal, lock_done);
2010 
2011     // Test if the oopMark is an obvious stack pointer, i.e.,
2012     //  1) (mark & 3) == 0, and
2013     //  2) rsp <= mark < mark + os::pagesize()
2014     // These 3 tests can be done by evaluating the following
2015     // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2016     // assuming both stack pointer and pagesize have their
2017     // least significant 2 bits clear.
2018     // NOTE: the oopMark is in swap_reg %rax, as the result of cmpxchg
2019 
2020     __ subptr(swap_reg, rsp);
2021     __ andptr(swap_reg, 3 - os::vm_page_size());
2022 
2023     // Save the test result, for recursive case, the result is zero
2024     __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2025     __ jcc(Assembler::notEqual, slow_path_lock);
2026     // Slow path will re-enter here
2027     __ bind(lock_done);
2028 
2029     if (UseBiasedLocking) {
2030       // Re-fetch oop_handle_reg as we trashed it above
2031       __ movptr(oop_handle_reg, Address(rsp, wordSize));
2032     }
2033   }
2034 
2035 
2036   // Finally just about ready to make the JNI call
2037 
2038 
2039   // get JNIEnv* which is first argument to native
2040   if (!is_critical_native) {
2041     __ lea(rdx, Address(thread, in_bytes(JavaThread::jni_environment_offset())));
2042     __ movptr(Address(rsp, 0), rdx);
2043   }
2044 
2045   // Now set thread in native
2046   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native);
2047 
2048   __ call(RuntimeAddress(native_func));
2049 
2050   // Verify or restore cpu control state after JNI call
2051   __ restore_cpu_control_state_after_jni();
2052 
2053   // WARNING - on Windows Java Natives use pascal calling convention and pop the
2054   // arguments off of the stack. We could just re-adjust the stack pointer here
2055   // and continue to do SP relative addressing but we instead switch to FP
2056   // relative addressing.
2057 
2058   // Unpack native results.
2059   switch (ret_type) {
2060   case T_BOOLEAN: __ c2bool(rax);            break;
2061   case T_CHAR   : __ andptr(rax, 0xFFFF);    break;
2062   case T_BYTE   : __ sign_extend_byte (rax); break;
2063   case T_SHORT  : __ sign_extend_short(rax); break;
2064   case T_INT    : /* nothing to do */        break;
2065   case T_DOUBLE :
2066   case T_FLOAT  :
2067     // Result is in st0 we'll save as needed
2068     break;
2069   case T_ARRAY:                 // Really a handle
2070   case T_OBJECT:                // Really a handle
2071       break; // can't de-handlize until after safepoint check
2072   case T_VOID: break;
2073   case T_LONG: break;
2074   default       : ShouldNotReachHere();
2075   }
2076 
2077   // Switch thread to "native transition" state before reading the synchronization state.
2078   // This additional state is necessary because reading and testing the synchronization
2079   // state is not atomic w.r.t. GC, as this scenario demonstrates:
2080   //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2081   //     VM thread changes sync state to synchronizing and suspends threads for GC.
2082   //     Thread A is resumed to finish this native method, but doesn't block here since it
2083   //     didn't see any synchronization is progress, and escapes.
2084   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2085 
2086   if(os::is_MP()) {
2087     if (UseMembar) {
2088       // Force this write out before the read below
2089       __ membar(Assembler::Membar_mask_bits(
2090            Assembler::LoadLoad | Assembler::LoadStore |
2091            Assembler::StoreLoad | Assembler::StoreStore));
2092     } else {
2093       // Write serialization page so VM thread can do a pseudo remote membar.
2094       // We use the current thread pointer to calculate a thread specific
2095       // offset to write to within the page. This minimizes bus traffic
2096       // due to cache line collision.
2097       __ serialize_memory(thread, rcx);
2098     }
2099   }
2100 
2101   if (AlwaysRestoreFPU) {
2102     // Make sure the control word is correct.
2103     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
2104   }
2105 
2106   Label after_transition;
2107 
2108   // check for safepoint operation in progress and/or pending suspend requests
2109   { Label Continue;
2110 
2111     __ cmp32(ExternalAddress((address)SafepointSynchronize::address_of_state()),
2112              SafepointSynchronize::_not_synchronized);
2113 
2114     Label L;
2115     __ jcc(Assembler::notEqual, L);
2116     __ cmpl(Address(thread, JavaThread::suspend_flags_offset()), 0);
2117     __ jcc(Assembler::equal, Continue);
2118     __ bind(L);
2119 
2120     // Don't use call_VM as it will see a possible pending exception and forward it
2121     // and never return here preventing us from clearing _last_native_pc down below.
2122     // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
2123     // preserved and correspond to the bcp/locals pointers. So we do a runtime call
2124     // by hand.
2125     //
2126     save_native_result(masm, ret_type, stack_slots);
2127     __ push(thread);
2128     if (!is_critical_native) {
2129       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address,
2130                                               JavaThread::check_special_condition_for_native_trans)));
2131     } else {
2132       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address,
2133                                               JavaThread::check_special_condition_for_native_trans_and_transition)));
2134     }
2135     __ increment(rsp, wordSize);
2136     // Restore any method result value
2137     restore_native_result(masm, ret_type, stack_slots);
2138 
2139     if (is_critical_native) {
2140       // The call above performed the transition to thread_in_Java so
2141       // skip the transition logic below.
2142       __ jmpb(after_transition);
2143     }
2144 
2145     __ bind(Continue);
2146   }
2147 
2148   // change thread state
2149   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_Java);
2150   __ bind(after_transition);
2151 
2152   Label reguard;
2153   Label reguard_done;
2154   __ cmpl(Address(thread, JavaThread::stack_guard_state_offset()), JavaThread::stack_guard_yellow_reserved_disabled);
2155   __ jcc(Assembler::equal, reguard);
2156 
2157   // slow path reguard  re-enters here
2158   __ bind(reguard_done);
2159 
2160   // Handle possible exception (will unlock if necessary)
2161 
2162   // native result if any is live
2163 
2164   // Unlock
2165   Label slow_path_unlock;
2166   Label unlock_done;
2167   if (method->is_synchronized()) {
2168 
2169     Label done;
2170 
2171     // Get locked oop from the handle we passed to jni
2172     __ movptr(obj_reg, Address(oop_handle_reg, 0));
2173 
2174     if (UseBiasedLocking) {
2175       __ biased_locking_exit(obj_reg, rbx, done);
2176     }
2177 
2178     // Simple recursive lock?
2179 
2180     __ cmpptr(Address(rbp, lock_slot_rbp_offset), (int32_t)NULL_WORD);
2181     __ jcc(Assembler::equal, done);
2182 
2183     // Must save rax, if if it is live now because cmpxchg must use it
2184     if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2185       save_native_result(masm, ret_type, stack_slots);
2186     }
2187 
2188     //  get old displaced header
2189     __ movptr(rbx, Address(rbp, lock_slot_rbp_offset));
2190 
2191     // get address of the stack lock
2192     __ lea(rax, Address(rbp, lock_slot_rbp_offset));
2193 
2194     // Atomic swap old header if oop still contains the stack lock
2195     if (os::is_MP()) {
2196     __ lock();
2197     }
2198 
2199     // src -> dest iff dest == rax, else rax, <- dest
2200     // *obj_reg = rbx, iff *obj_reg == rax, else rax, = *(obj_reg)
2201     __ cmpxchgptr(rbx, Address(obj_reg, 0));
2202     __ jcc(Assembler::notEqual, slow_path_unlock);
2203 
2204     // slow path re-enters here
2205     __ bind(unlock_done);
2206     if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2207       restore_native_result(masm, ret_type, stack_slots);
2208     }
2209 
2210     __ bind(done);
2211 
2212   }
2213 
2214   {
2215     SkipIfEqual skip_if(masm, &DTraceMethodProbes, 0);
2216     // Tell dtrace about this method exit
2217     save_native_result(masm, ret_type, stack_slots);
2218     __ mov_metadata(rax, method());
2219     __ call_VM_leaf(
2220          CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2221          thread, rax);
2222     restore_native_result(masm, ret_type, stack_slots);
2223   }
2224 
2225   // We can finally stop using that last_Java_frame we setup ages ago
2226 
2227   __ reset_last_Java_frame(thread, false, true);
2228 
2229   // Unpack oop result
2230   if (ret_type == T_OBJECT || ret_type == T_ARRAY) {
2231       Label L;
2232       __ cmpptr(rax, (int32_t)NULL_WORD);
2233       __ jcc(Assembler::equal, L);
2234       __ movptr(rax, Address(rax, 0));
2235       __ bind(L);
2236       __ verify_oop(rax);
2237   }
2238 
2239   if (!is_critical_native) {
2240     // reset handle block
2241     __ movptr(rcx, Address(thread, JavaThread::active_handles_offset()));
2242     __ movl(Address(rcx, JNIHandleBlock::top_offset_in_bytes()), NULL_WORD);
2243 
2244     // Any exception pending?
2245     __ cmpptr(Address(thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD);
2246     __ jcc(Assembler::notEqual, exception_pending);
2247   }
2248 
2249   // no exception, we're almost done
2250 
2251   // check that only result value is on FPU stack
2252   __ verify_FPU(ret_type == T_FLOAT || ret_type == T_DOUBLE ? 1 : 0, "native_wrapper normal exit");
2253 
2254   // Fixup floating pointer results so that result looks like a return from a compiled method
2255   if (ret_type == T_FLOAT) {
2256     if (UseSSE >= 1) {
2257       // Pop st0 and store as float and reload into xmm register
2258       __ fstp_s(Address(rbp, -4));
2259       __ movflt(xmm0, Address(rbp, -4));
2260     }
2261   } else if (ret_type == T_DOUBLE) {
2262     if (UseSSE >= 2) {
2263       // Pop st0 and store as double and reload into xmm register
2264       __ fstp_d(Address(rbp, -8));
2265       __ movdbl(xmm0, Address(rbp, -8));
2266     }
2267   }
2268 
2269   // Return
2270 
2271   __ leave();
2272   __ ret(0);
2273 
2274   // Unexpected paths are out of line and go here
2275 
2276   // Slow path locking & unlocking
2277   if (method->is_synchronized()) {
2278 
2279     // BEGIN Slow path lock
2280 
2281     __ bind(slow_path_lock);
2282 
2283     // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
2284     // args are (oop obj, BasicLock* lock, JavaThread* thread)
2285     __ push(thread);
2286     __ push(lock_reg);
2287     __ push(obj_reg);
2288     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C)));
2289     __ addptr(rsp, 3*wordSize);
2290 
2291 #ifdef ASSERT
2292     { Label L;
2293     __ cmpptr(Address(thread, in_bytes(Thread::pending_exception_offset())), (int)NULL_WORD);
2294     __ jcc(Assembler::equal, L);
2295     __ stop("no pending exception allowed on exit from monitorenter");
2296     __ bind(L);
2297     }
2298 #endif
2299     __ jmp(lock_done);
2300 
2301     // END Slow path lock
2302 
2303     // BEGIN Slow path unlock
2304     __ bind(slow_path_unlock);
2305 
2306     // Slow path unlock
2307 
2308     if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
2309       save_native_result(masm, ret_type, stack_slots);
2310     }
2311     // Save pending exception around call to VM (which contains an EXCEPTION_MARK)
2312 
2313     __ pushptr(Address(thread, in_bytes(Thread::pending_exception_offset())));
2314     __ movptr(Address(thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD);
2315 
2316 
2317     // should be a peal
2318     // +wordSize because of the push above
2319     // args are (oop obj, BasicLock* lock, JavaThread* thread)
2320     __ push(thread);
2321     __ lea(rax, Address(rbp, lock_slot_rbp_offset));
2322     __ push(rax);
2323 
2324     __ push(obj_reg);
2325     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C)));
2326     __ addptr(rsp, 3*wordSize);
2327 #ifdef ASSERT
2328     {
2329       Label L;
2330       __ cmpptr(Address(thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD);
2331       __ jcc(Assembler::equal, L);
2332       __ stop("no pending exception allowed on exit complete_monitor_unlocking_C");
2333       __ bind(L);
2334     }
2335 #endif /* ASSERT */
2336 
2337     __ popptr(Address(thread, in_bytes(Thread::pending_exception_offset())));
2338 
2339     if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
2340       restore_native_result(masm, ret_type, stack_slots);
2341     }
2342     __ jmp(unlock_done);
2343     // END Slow path unlock
2344 
2345   }
2346 
2347   // SLOW PATH Reguard the stack if needed
2348 
2349   __ bind(reguard);
2350   save_native_result(masm, ret_type, stack_slots);
2351   {
2352     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
2353   }
2354   restore_native_result(masm, ret_type, stack_slots);
2355   __ jmp(reguard_done);
2356 
2357 
2358   // BEGIN EXCEPTION PROCESSING
2359 
2360   if (!is_critical_native) {
2361     // Forward  the exception
2362     __ bind(exception_pending);
2363 
2364     // remove possible return value from FPU register stack
2365     __ empty_FPU_stack();
2366 
2367     // pop our frame
2368     __ leave();
2369     // and forward the exception
2370     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2371   }
2372 
2373   __ flush();
2374 
2375   nmethod *nm = nmethod::new_native_nmethod(method,
2376                                             compile_id,
2377                                             masm->code(),
2378                                             vep_offset,
2379                                             frame_complete,
2380                                             stack_slots / VMRegImpl::slots_per_word,
2381                                             (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2382                                             in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
2383                                             oop_maps);
2384 
2385   if (is_critical_native) {
2386     nm->set_lazy_critical_native(true);
2387   }
2388 
2389   return nm;
2390 
2391 }
2392 
2393 // this function returns the adjust size (in number of words) to a c2i adapter
2394 // activation for use during deoptimization
2395 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals ) {
2396   return (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2397 }
2398 
2399 
2400 uint SharedRuntime::out_preserve_stack_slots() {
2401   return 0;
2402 }
2403 
2404 //------------------------------generate_deopt_blob----------------------------
2405 void SharedRuntime::generate_deopt_blob() {
2406   // allocate space for the code
2407   ResourceMark rm;
2408   // setup code generation tools
2409   // note: the buffer code size must account for StackShadowPages=50
2410   CodeBuffer   buffer("deopt_blob", 1536, 1024);
2411   MacroAssembler* masm = new MacroAssembler(&buffer);
2412   int frame_size_in_words;
2413   OopMap* map = NULL;
2414   // Account for the extra args we place on the stack
2415   // by the time we call fetch_unroll_info
2416   const int additional_words = 2; // deopt kind, thread
2417 
2418   OopMapSet *oop_maps = new OopMapSet();
2419 
2420   // -------------
2421   // This code enters when returning to a de-optimized nmethod.  A return
2422   // address has been pushed on the the stack, and return values are in
2423   // registers.
2424   // If we are doing a normal deopt then we were called from the patched
2425   // nmethod from the point we returned to the nmethod. So the return
2426   // address on the stack is wrong by NativeCall::instruction_size
2427   // We will adjust the value to it looks like we have the original return
2428   // address on the stack (like when we eagerly deoptimized).
2429   // In the case of an exception pending with deoptimized then we enter
2430   // with a return address on the stack that points after the call we patched
2431   // into the exception handler. We have the following register state:
2432   //    rax,: exception
2433   //    rbx,: exception handler
2434   //    rdx: throwing pc
2435   // So in this case we simply jam rdx into the useless return address and
2436   // the stack looks just like we want.
2437   //
2438   // At this point we need to de-opt.  We save the argument return
2439   // registers.  We call the first C routine, fetch_unroll_info().  This
2440   // routine captures the return values and returns a structure which
2441   // describes the current frame size and the sizes of all replacement frames.
2442   // The current frame is compiled code and may contain many inlined
2443   // functions, each with their own JVM state.  We pop the current frame, then
2444   // push all the new frames.  Then we call the C routine unpack_frames() to
2445   // populate these frames.  Finally unpack_frames() returns us the new target
2446   // address.  Notice that callee-save registers are BLOWN here; they have
2447   // already been captured in the vframeArray at the time the return PC was
2448   // patched.
2449   address start = __ pc();
2450   Label cont;
2451 
2452   // Prolog for non exception case!
2453 
2454   // Save everything in sight.
2455 
2456   map = RegisterSaver::save_live_registers(masm, additional_words, &frame_size_in_words, false);
2457   // Normal deoptimization
2458   __ push(Deoptimization::Unpack_deopt);
2459   __ jmp(cont);
2460 
2461   int reexecute_offset = __ pc() - start;
2462 
2463   // Reexecute case
2464   // return address is the pc describes what bci to do re-execute at
2465 
2466   // No need to update map as each call to save_live_registers will produce identical oopmap
2467   (void) RegisterSaver::save_live_registers(masm, additional_words, &frame_size_in_words, false);
2468 
2469   __ push(Deoptimization::Unpack_reexecute);
2470   __ jmp(cont);
2471 
2472   int exception_offset = __ pc() - start;
2473 
2474   // Prolog for exception case
2475 
2476   // all registers are dead at this entry point, except for rax, and
2477   // rdx which contain the exception oop and exception pc
2478   // respectively.  Set them in TLS and fall thru to the
2479   // unpack_with_exception_in_tls entry point.
2480 
2481   __ get_thread(rdi);
2482   __ movptr(Address(rdi, JavaThread::exception_pc_offset()), rdx);
2483   __ movptr(Address(rdi, JavaThread::exception_oop_offset()), rax);
2484 
2485   int exception_in_tls_offset = __ pc() - start;
2486 
2487   // new implementation because exception oop is now passed in JavaThread
2488 
2489   // Prolog for exception case
2490   // All registers must be preserved because they might be used by LinearScan
2491   // Exceptiop oop and throwing PC are passed in JavaThread
2492   // tos: stack at point of call to method that threw the exception (i.e. only
2493   // args are on the stack, no return address)
2494 
2495   // make room on stack for the return address
2496   // It will be patched later with the throwing pc. The correct value is not
2497   // available now because loading it from memory would destroy registers.
2498   __ push(0);
2499 
2500   // Save everything in sight.
2501 
2502   // No need to update map as each call to save_live_registers will produce identical oopmap
2503   (void) RegisterSaver::save_live_registers(masm, additional_words, &frame_size_in_words, false);
2504 
2505   // Now it is safe to overwrite any register
2506 
2507   // store the correct deoptimization type
2508   __ push(Deoptimization::Unpack_exception);
2509 
2510   // load throwing pc from JavaThread and patch it as the return address
2511   // of the current frame. Then clear the field in JavaThread
2512   __ get_thread(rdi);
2513   __ movptr(rdx, Address(rdi, JavaThread::exception_pc_offset()));
2514   __ movptr(Address(rbp, wordSize), rdx);
2515   __ movptr(Address(rdi, JavaThread::exception_pc_offset()), NULL_WORD);
2516 
2517 #ifdef ASSERT
2518   // verify that there is really an exception oop in JavaThread
2519   __ movptr(rax, Address(rdi, JavaThread::exception_oop_offset()));
2520   __ verify_oop(rax);
2521 
2522   // verify that there is no pending exception
2523   Label no_pending_exception;
2524   __ movptr(rax, Address(rdi, Thread::pending_exception_offset()));
2525   __ testptr(rax, rax);
2526   __ jcc(Assembler::zero, no_pending_exception);
2527   __ stop("must not have pending exception here");
2528   __ bind(no_pending_exception);
2529 #endif
2530 
2531   __ bind(cont);
2532 
2533   // Compiled code leaves the floating point stack dirty, empty it.
2534   __ empty_FPU_stack();
2535 
2536 
2537   // Call C code.  Need thread and this frame, but NOT official VM entry
2538   // crud.  We cannot block on this call, no GC can happen.
2539   __ get_thread(rcx);
2540   __ push(rcx);
2541   // fetch_unroll_info needs to call last_java_frame()
2542   __ set_last_Java_frame(rcx, noreg, noreg, NULL);
2543 
2544   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
2545 
2546   // Need to have an oopmap that tells fetch_unroll_info where to
2547   // find any register it might need.
2548 
2549   oop_maps->add_gc_map( __ pc()-start, map);
2550 
2551   // Discard args to fetch_unroll_info
2552   __ pop(rcx);
2553   __ pop(rcx);
2554 
2555   __ get_thread(rcx);
2556   __ reset_last_Java_frame(rcx, false, false);
2557 
2558   // Load UnrollBlock into EDI
2559   __ mov(rdi, rax);
2560 
2561   // Move the unpack kind to a safe place in the UnrollBlock because
2562   // we are very short of registers
2563 
2564   Address unpack_kind(rdi, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes());
2565   // retrieve the deopt kind from the UnrollBlock.
2566   __ movl(rax, unpack_kind);
2567 
2568    Label noException;
2569   __ cmpl(rax, Deoptimization::Unpack_exception);   // Was exception pending?
2570   __ jcc(Assembler::notEqual, noException);
2571   __ movptr(rax, Address(rcx, JavaThread::exception_oop_offset()));
2572   __ movptr(rdx, Address(rcx, JavaThread::exception_pc_offset()));
2573   __ movptr(Address(rcx, JavaThread::exception_oop_offset()), NULL_WORD);
2574   __ movptr(Address(rcx, JavaThread::exception_pc_offset()), NULL_WORD);
2575 
2576   __ verify_oop(rax);
2577 
2578   // Overwrite the result registers with the exception results.
2579   __ movptr(Address(rsp, RegisterSaver::raxOffset()*wordSize), rax);
2580   __ movptr(Address(rsp, RegisterSaver::rdxOffset()*wordSize), rdx);
2581 
2582   __ bind(noException);
2583 
2584   // Stack is back to only having register save data on the stack.
2585   // Now restore the result registers. Everything else is either dead or captured
2586   // in the vframeArray.
2587 
2588   RegisterSaver::restore_result_registers(masm);
2589 
2590   // Non standard control word may be leaked out through a safepoint blob, and we can
2591   // deopt at a poll point with the non standard control word. However, we should make
2592   // sure the control word is correct after restore_result_registers.
2593   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
2594 
2595   // All of the register save area has been popped of the stack. Only the
2596   // return address remains.
2597 
2598   // Pop all the frames we must move/replace.
2599   //
2600   // Frame picture (youngest to oldest)
2601   // 1: self-frame (no frame link)
2602   // 2: deopting frame  (no frame link)
2603   // 3: caller of deopting frame (could be compiled/interpreted).
2604   //
2605   // Note: by leaving the return address of self-frame on the stack
2606   // and using the size of frame 2 to adjust the stack
2607   // when we are done the return to frame 3 will still be on the stack.
2608 
2609   // Pop deoptimized frame
2610   __ addptr(rsp, Address(rdi,Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes()));
2611 
2612   // sp should be pointing at the return address to the caller (3)
2613 
2614   // Pick up the initial fp we should save
2615   // restore rbp before stack bang because if stack overflow is thrown it needs to be pushed (and preserved)
2616   __ movptr(rbp, Address(rdi, Deoptimization::UnrollBlock::initial_info_offset_in_bytes()));
2617 
2618 #ifdef ASSERT
2619   // Compilers generate code that bang the stack by as much as the
2620   // interpreter would need. So this stack banging should never
2621   // trigger a fault. Verify that it does not on non product builds.
2622   if (UseStackBanging) {
2623     __ movl(rbx, Address(rdi ,Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
2624     __ bang_stack_size(rbx, rcx);
2625   }
2626 #endif
2627 
2628   // Load array of frame pcs into ECX
2629   __ movptr(rcx,Address(rdi,Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2630 
2631   __ pop(rsi); // trash the old pc
2632 
2633   // Load array of frame sizes into ESI
2634   __ movptr(rsi,Address(rdi,Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes()));
2635 
2636   Address counter(rdi, Deoptimization::UnrollBlock::counter_temp_offset_in_bytes());
2637 
2638   __ movl(rbx, Address(rdi, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes()));
2639   __ movl(counter, rbx);
2640 
2641   // Now adjust the caller's stack to make up for the extra locals
2642   // but record the original sp so that we can save it in the skeletal interpreter
2643   // frame and the stack walking of interpreter_sender will get the unextended sp
2644   // value and not the "real" sp value.
2645 
2646   Address sp_temp(rdi, Deoptimization::UnrollBlock::sender_sp_temp_offset_in_bytes());
2647   __ movptr(sp_temp, rsp);
2648   __ movl2ptr(rbx, Address(rdi, Deoptimization::UnrollBlock::caller_adjustment_offset_in_bytes()));
2649   __ subptr(rsp, rbx);
2650 
2651   // Push interpreter frames in a loop
2652   Label loop;
2653   __ bind(loop);
2654   __ movptr(rbx, Address(rsi, 0));      // Load frame size
2655   __ subptr(rbx, 2*wordSize);           // we'll push pc and rbp, by hand
2656   __ pushptr(Address(rcx, 0));          // save return address
2657   __ enter();                           // save old & set new rbp,
2658   __ subptr(rsp, rbx);                  // Prolog!
2659   __ movptr(rbx, sp_temp);              // sender's sp
2660   // This value is corrected by layout_activation_impl
2661   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
2662   __ movptr(Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize), rbx); // Make it walkable
2663   __ movptr(sp_temp, rsp);              // pass to next frame
2664   __ addptr(rsi, wordSize);             // Bump array pointer (sizes)
2665   __ addptr(rcx, wordSize);             // Bump array pointer (pcs)
2666   __ decrementl(counter);             // decrement counter
2667   __ jcc(Assembler::notZero, loop);
2668   __ pushptr(Address(rcx, 0));          // save final return address
2669 
2670   // Re-push self-frame
2671   __ enter();                           // save old & set new rbp,
2672 
2673   //  Return address and rbp, are in place
2674   // We'll push additional args later. Just allocate a full sized
2675   // register save area
2676   __ subptr(rsp, (frame_size_in_words-additional_words - 2) * wordSize);
2677 
2678   // Restore frame locals after moving the frame
2679   __ movptr(Address(rsp, RegisterSaver::raxOffset()*wordSize), rax);
2680   __ movptr(Address(rsp, RegisterSaver::rdxOffset()*wordSize), rdx);
2681   __ fstp_d(Address(rsp, RegisterSaver::fpResultOffset()*wordSize));   // Pop float stack and store in local
2682   if( UseSSE>=2 ) __ movdbl(Address(rsp, RegisterSaver::xmm0Offset()*wordSize), xmm0);
2683   if( UseSSE==1 ) __ movflt(Address(rsp, RegisterSaver::xmm0Offset()*wordSize), xmm0);
2684 
2685   // Set up the args to unpack_frame
2686 
2687   __ pushl(unpack_kind);                     // get the unpack_kind value
2688   __ get_thread(rcx);
2689   __ push(rcx);
2690 
2691   // set last_Java_sp, last_Java_fp
2692   __ set_last_Java_frame(rcx, noreg, rbp, NULL);
2693 
2694   // Call C code.  Need thread but NOT official VM entry
2695   // crud.  We cannot block on this call, no GC can happen.  Call should
2696   // restore return values to their stack-slots with the new SP.
2697   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2698   // Set an oopmap for the call site
2699   oop_maps->add_gc_map( __ pc()-start, new OopMap( frame_size_in_words, 0 ));
2700 
2701   // rax, contains the return result type
2702   __ push(rax);
2703 
2704   __ get_thread(rcx);
2705   __ reset_last_Java_frame(rcx, false, false);
2706 
2707   // Collect return values
2708   __ movptr(rax,Address(rsp, (RegisterSaver::raxOffset() + additional_words + 1)*wordSize));
2709   __ movptr(rdx,Address(rsp, (RegisterSaver::rdxOffset() + additional_words + 1)*wordSize));
2710 
2711   // Clear floating point stack before returning to interpreter
2712   __ empty_FPU_stack();
2713 
2714   // Check if we should push the float or double return value.
2715   Label results_done, yes_double_value;
2716   __ cmpl(Address(rsp, 0), T_DOUBLE);
2717   __ jcc (Assembler::zero, yes_double_value);
2718   __ cmpl(Address(rsp, 0), T_FLOAT);
2719   __ jcc (Assembler::notZero, results_done);
2720 
2721   // return float value as expected by interpreter
2722   if( UseSSE>=1 ) __ movflt(xmm0, Address(rsp, (RegisterSaver::xmm0Offset() + additional_words + 1)*wordSize));
2723   else            __ fld_d(Address(rsp, (RegisterSaver::fpResultOffset() + additional_words + 1)*wordSize));
2724   __ jmp(results_done);
2725 
2726   // return double value as expected by interpreter
2727   __ bind(yes_double_value);
2728   if( UseSSE>=2 ) __ movdbl(xmm0, Address(rsp, (RegisterSaver::xmm0Offset() + additional_words + 1)*wordSize));
2729   else            __ fld_d(Address(rsp, (RegisterSaver::fpResultOffset() + additional_words + 1)*wordSize));
2730 
2731   __ bind(results_done);
2732 
2733   // Pop self-frame.
2734   __ leave();                              // Epilog!
2735 
2736   // Jump to interpreter
2737   __ ret(0);
2738 
2739   // -------------
2740   // make sure all code is generated
2741   masm->flush();
2742 
2743   _deopt_blob = DeoptimizationBlob::create( &buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
2744   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
2745 }
2746 
2747 
2748 #ifdef COMPILER2
2749 //------------------------------generate_uncommon_trap_blob--------------------
2750 void SharedRuntime::generate_uncommon_trap_blob() {
2751   // allocate space for the code
2752   ResourceMark rm;
2753   // setup code generation tools
2754   CodeBuffer   buffer("uncommon_trap_blob", 512, 512);
2755   MacroAssembler* masm = new MacroAssembler(&buffer);
2756 
2757   enum frame_layout {
2758     arg0_off,      // thread                     sp + 0 // Arg location for
2759     arg1_off,      // unloaded_class_index       sp + 1 // calling C
2760     arg2_off,      // exec_mode                  sp + 2
2761     // The frame sender code expects that rbp will be in the "natural" place and
2762     // will override any oopMap setting for it. We must therefore force the layout
2763     // so that it agrees with the frame sender code.
2764     rbp_off,       // callee saved register      sp + 3
2765     return_off,    // slot for return address    sp + 4
2766     framesize
2767   };
2768 
2769   address start = __ pc();
2770 
2771   if (UseRTMLocking) {
2772     // Abort RTM transaction before possible nmethod deoptimization.
2773     __ xabort(0);
2774   }
2775 
2776   // Push self-frame.
2777   __ subptr(rsp, return_off*wordSize);     // Epilog!
2778 
2779   // rbp, is an implicitly saved callee saved register (i.e. the calling
2780   // convention will save restore it in prolog/epilog) Other than that
2781   // there are no callee save registers no that adapter frames are gone.
2782   __ movptr(Address(rsp, rbp_off*wordSize), rbp);
2783 
2784   // Clear the floating point exception stack
2785   __ empty_FPU_stack();
2786 
2787   // set last_Java_sp
2788   __ get_thread(rdx);
2789   __ set_last_Java_frame(rdx, noreg, noreg, NULL);
2790 
2791   // Call C code.  Need thread but NOT official VM entry
2792   // crud.  We cannot block on this call, no GC can happen.  Call should
2793   // capture callee-saved registers as well as return values.
2794   __ movptr(Address(rsp, arg0_off*wordSize), rdx);
2795   // argument already in ECX
2796   __ movl(Address(rsp, arg1_off*wordSize),rcx);
2797   __ movl(Address(rsp, arg2_off*wordSize), Deoptimization::Unpack_uncommon_trap);
2798   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap)));
2799 
2800   // Set an oopmap for the call site
2801   OopMapSet *oop_maps = new OopMapSet();
2802   OopMap* map =  new OopMap( framesize, 0 );
2803   // No oopMap for rbp, it is known implicitly
2804 
2805   oop_maps->add_gc_map( __ pc()-start, map);
2806 
2807   __ get_thread(rcx);
2808 
2809   __ reset_last_Java_frame(rcx, false, false);
2810 
2811   // Load UnrollBlock into EDI
2812   __ movptr(rdi, rax);
2813 
2814 #ifdef ASSERT
2815   { Label L;
2816     __ cmpptr(Address(rdi, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()),
2817             (int32_t)Deoptimization::Unpack_uncommon_trap);
2818     __ jcc(Assembler::equal, L);
2819     __ stop("SharedRuntime::generate_deopt_blob: expected Unpack_uncommon_trap");
2820     __ bind(L);
2821   }
2822 #endif
2823 
2824   // Pop all the frames we must move/replace.
2825   //
2826   // Frame picture (youngest to oldest)
2827   // 1: self-frame (no frame link)
2828   // 2: deopting frame  (no frame link)
2829   // 3: caller of deopting frame (could be compiled/interpreted).
2830 
2831   // Pop self-frame.  We have no frame, and must rely only on EAX and ESP.
2832   __ addptr(rsp,(framesize-1)*wordSize);     // Epilog!
2833 
2834   // Pop deoptimized frame
2835   __ movl2ptr(rcx, Address(rdi,Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes()));
2836   __ addptr(rsp, rcx);
2837 
2838   // sp should be pointing at the return address to the caller (3)
2839 
2840   // Pick up the initial fp we should save
2841   // restore rbp before stack bang because if stack overflow is thrown it needs to be pushed (and preserved)
2842   __ movptr(rbp, Address(rdi, Deoptimization::UnrollBlock::initial_info_offset_in_bytes()));
2843 
2844 #ifdef ASSERT
2845   // Compilers generate code that bang the stack by as much as the
2846   // interpreter would need. So this stack banging should never
2847   // trigger a fault. Verify that it does not on non product builds.
2848   if (UseStackBanging) {
2849     __ movl(rbx, Address(rdi ,Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
2850     __ bang_stack_size(rbx, rcx);
2851   }
2852 #endif
2853 
2854   // Load array of frame pcs into ECX
2855   __ movl(rcx,Address(rdi,Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2856 
2857   __ pop(rsi); // trash the pc
2858 
2859   // Load array of frame sizes into ESI
2860   __ movptr(rsi,Address(rdi,Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes()));
2861 
2862   Address counter(rdi, Deoptimization::UnrollBlock::counter_temp_offset_in_bytes());
2863 
2864   __ movl(rbx, Address(rdi, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes()));
2865   __ movl(counter, rbx);
2866 
2867   // Now adjust the caller's stack to make up for the extra locals
2868   // but record the original sp so that we can save it in the skeletal interpreter
2869   // frame and the stack walking of interpreter_sender will get the unextended sp
2870   // value and not the "real" sp value.
2871 
2872   Address sp_temp(rdi, Deoptimization::UnrollBlock::sender_sp_temp_offset_in_bytes());
2873   __ movptr(sp_temp, rsp);
2874   __ movl(rbx, Address(rdi, Deoptimization::UnrollBlock::caller_adjustment_offset_in_bytes()));
2875   __ subptr(rsp, rbx);
2876 
2877   // Push interpreter frames in a loop
2878   Label loop;
2879   __ bind(loop);
2880   __ movptr(rbx, Address(rsi, 0));      // Load frame size
2881   __ subptr(rbx, 2*wordSize);           // we'll push pc and rbp, by hand
2882   __ pushptr(Address(rcx, 0));          // save return address
2883   __ enter();                           // save old & set new rbp,
2884   __ subptr(rsp, rbx);                  // Prolog!
2885   __ movptr(rbx, sp_temp);              // sender's sp
2886   // This value is corrected by layout_activation_impl
2887   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD );
2888   __ movptr(Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize), rbx); // Make it walkable
2889   __ movptr(sp_temp, rsp);              // pass to next frame
2890   __ addptr(rsi, wordSize);             // Bump array pointer (sizes)
2891   __ addptr(rcx, wordSize);             // Bump array pointer (pcs)
2892   __ decrementl(counter);             // decrement counter
2893   __ jcc(Assembler::notZero, loop);
2894   __ pushptr(Address(rcx, 0));            // save final return address
2895 
2896   // Re-push self-frame
2897   __ enter();                           // save old & set new rbp,
2898   __ subptr(rsp, (framesize-2) * wordSize);   // Prolog!
2899 
2900 
2901   // set last_Java_sp, last_Java_fp
2902   __ get_thread(rdi);
2903   __ set_last_Java_frame(rdi, noreg, rbp, NULL);
2904 
2905   // Call C code.  Need thread but NOT official VM entry
2906   // crud.  We cannot block on this call, no GC can happen.  Call should
2907   // restore return values to their stack-slots with the new SP.
2908   __ movptr(Address(rsp,arg0_off*wordSize),rdi);
2909   __ movl(Address(rsp,arg1_off*wordSize), Deoptimization::Unpack_uncommon_trap);
2910   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2911   // Set an oopmap for the call site
2912   oop_maps->add_gc_map( __ pc()-start, new OopMap( framesize, 0 ) );
2913 
2914   __ get_thread(rdi);
2915   __ reset_last_Java_frame(rdi, true, false);
2916 
2917   // Pop self-frame.
2918   __ leave();     // Epilog!
2919 
2920   // Jump to interpreter
2921   __ ret(0);
2922 
2923   // -------------
2924   // make sure all code is generated
2925   masm->flush();
2926 
2927    _uncommon_trap_blob = UncommonTrapBlob::create(&buffer, oop_maps, framesize);
2928 }
2929 #endif // COMPILER2
2930 
2931 //------------------------------generate_handler_blob------
2932 //
2933 // Generate a special Compile2Runtime blob that saves all registers,
2934 // setup oopmap, and calls safepoint code to stop the compiled code for
2935 // a safepoint.
2936 //
2937 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
2938 
2939   // Account for thread arg in our frame
2940   const int additional_words = 1;
2941   int frame_size_in_words;
2942 
2943   assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
2944 
2945   ResourceMark rm;
2946   OopMapSet *oop_maps = new OopMapSet();
2947   OopMap* map;
2948 
2949   // allocate space for the code
2950   // setup code generation tools
2951   CodeBuffer   buffer("handler_blob", 1024, 512);
2952   MacroAssembler* masm = new MacroAssembler(&buffer);
2953 
2954   const Register java_thread = rdi; // callee-saved for VC++
2955   address start   = __ pc();
2956   address call_pc = NULL;
2957   bool cause_return = (poll_type == POLL_AT_RETURN);
2958   bool save_vectors = (poll_type == POLL_AT_VECTOR_LOOP);
2959 
2960   if (UseRTMLocking) {
2961     // Abort RTM transaction before calling runtime
2962     // because critical section will be large and will be
2963     // aborted anyway. Also nmethod could be deoptimized.
2964     __ xabort(0);
2965   }
2966 
2967   // If cause_return is true we are at a poll_return and there is
2968   // the return address on the stack to the caller on the nmethod
2969   // that is safepoint. We can leave this return on the stack and
2970   // effectively complete the return and safepoint in the caller.
2971   // Otherwise we push space for a return address that the safepoint
2972   // handler will install later to make the stack walking sensible.
2973   if (!cause_return)
2974     __ push(rbx);  // Make room for return address (or push it again)
2975 
2976   map = RegisterSaver::save_live_registers(masm, additional_words, &frame_size_in_words, false, save_vectors);
2977 
2978   // The following is basically a call_VM. However, we need the precise
2979   // address of the call in order to generate an oopmap. Hence, we do all the
2980   // work ourselves.
2981 
2982   // Push thread argument and setup last_Java_sp
2983   __ get_thread(java_thread);
2984   __ push(java_thread);
2985   __ set_last_Java_frame(java_thread, noreg, noreg, NULL);
2986 
2987   // if this was not a poll_return then we need to correct the return address now.
2988   if (!cause_return) {
2989     __ movptr(rax, Address(java_thread, JavaThread::saved_exception_pc_offset()));
2990     __ movptr(Address(rbp, wordSize), rax);
2991   }
2992 
2993   // do the call
2994   __ call(RuntimeAddress(call_ptr));
2995 
2996   // Set an oopmap for the call site.  This oopmap will map all
2997   // oop-registers and debug-info registers as callee-saved.  This
2998   // will allow deoptimization at this safepoint to find all possible
2999   // debug-info recordings, as well as let GC find all oops.
3000 
3001   oop_maps->add_gc_map( __ pc() - start, map);
3002 
3003   // Discard arg
3004   __ pop(rcx);
3005 
3006   Label noException;
3007 
3008   // Clear last_Java_sp again
3009   __ get_thread(java_thread);
3010   __ reset_last_Java_frame(java_thread, false, false);
3011 
3012   __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3013   __ jcc(Assembler::equal, noException);
3014 
3015   // Exception pending
3016   RegisterSaver::restore_live_registers(masm, save_vectors);
3017 
3018   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3019 
3020   __ bind(noException);
3021 
3022   // Normal exit, register restoring and exit
3023   RegisterSaver::restore_live_registers(masm, save_vectors);
3024 
3025   __ ret(0);
3026 
3027   // make sure all code is generated
3028   masm->flush();
3029 
3030   // Fill-out other meta info
3031   return SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
3032 }
3033 
3034 //
3035 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss
3036 //
3037 // Generate a stub that calls into vm to find out the proper destination
3038 // of a java call. All the argument registers are live at this point
3039 // but since this is generic code we don't know what they are and the caller
3040 // must do any gc of the args.
3041 //
3042 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
3043   assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
3044 
3045   // allocate space for the code
3046   ResourceMark rm;
3047 
3048   CodeBuffer buffer(name, 1000, 512);
3049   MacroAssembler* masm                = new MacroAssembler(&buffer);
3050 
3051   int frame_size_words;
3052   enum frame_layout {
3053                 thread_off,
3054                 extra_words };
3055 
3056   OopMapSet *oop_maps = new OopMapSet();
3057   OopMap* map = NULL;
3058 
3059   int start = __ offset();
3060 
3061   map = RegisterSaver::save_live_registers(masm, extra_words, &frame_size_words);
3062 
3063   int frame_complete = __ offset();
3064 
3065   const Register thread = rdi;
3066   __ get_thread(rdi);
3067 
3068   __ push(thread);
3069   __ set_last_Java_frame(thread, noreg, rbp, NULL);
3070 
3071   __ call(RuntimeAddress(destination));
3072 
3073 
3074   // Set an oopmap for the call site.
3075   // We need this not only for callee-saved registers, but also for volatile
3076   // registers that the compiler might be keeping live across a safepoint.
3077 
3078   oop_maps->add_gc_map( __ offset() - start, map);
3079 
3080   // rax, contains the address we are going to jump to assuming no exception got installed
3081 
3082   __ addptr(rsp, wordSize);
3083 
3084   // clear last_Java_sp
3085   __ reset_last_Java_frame(thread, true, false);
3086   // check for pending exceptions
3087   Label pending;
3088   __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3089   __ jcc(Assembler::notEqual, pending);
3090 
3091   // get the returned Method*
3092   __ get_vm_result_2(rbx, thread);
3093   __ movptr(Address(rsp, RegisterSaver::rbx_offset() * wordSize), rbx);
3094 
3095   __ movptr(Address(rsp, RegisterSaver::rax_offset() * wordSize), rax);
3096 
3097   RegisterSaver::restore_live_registers(masm);
3098 
3099   // We are back the the original state on entry and ready to go.
3100 
3101   __ jmp(rax);
3102 
3103   // Pending exception after the safepoint
3104 
3105   __ bind(pending);
3106 
3107   RegisterSaver::restore_live_registers(masm);
3108 
3109   // exception pending => remove activation and forward to exception handler
3110 
3111   __ get_thread(thread);
3112   __ movptr(Address(thread, JavaThread::vm_result_offset()), NULL_WORD);
3113   __ movptr(rax, Address(thread, Thread::pending_exception_offset()));
3114   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3115 
3116   // -------------
3117   // make sure all code is generated
3118   masm->flush();
3119 
3120   // return the  blob
3121   // frame_size_words or bytes??
3122   return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_words, oop_maps, true);
3123 }