1 /*
   2  * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright 2012, 2015 SAP AG. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "code/debugInfoRec.hpp"
  29 #include "code/icBuffer.hpp"
  30 #include "code/vtableStubs.hpp"
  31 #include "frame_ppc.hpp"
  32 #include "interpreter/interpreter.hpp"
  33 #include "interpreter/interp_masm.hpp"
  34 #include "oops/compiledICHolder.hpp"
  35 #include "prims/jvmtiRedefineClassesTrace.hpp"
  36 #include "runtime/sharedRuntime.hpp"
  37 #include "runtime/vframeArray.hpp"
  38 #include "vmreg_ppc.inline.hpp"
  39 #ifdef COMPILER1
  40 #include "c1/c1_Runtime1.hpp"
  41 #endif
  42 #ifdef COMPILER2
  43 #include "adfiles/ad_ppc_64.hpp"
  44 #include "opto/runtime.hpp"
  45 #endif
  46 
  47 #define __ masm->
  48 
  49 #ifdef PRODUCT
  50 #define BLOCK_COMMENT(str) // nothing
  51 #else
  52 #define BLOCK_COMMENT(str) __ block_comment(str)
  53 #endif
  54 
  55 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  56 
  57 
  58 class RegisterSaver {
  59  // Used for saving volatile registers.
  60  public:
  61 
  62   // Support different return pc locations.
  63   enum ReturnPCLocation {
  64     return_pc_is_lr,
  65     return_pc_is_r4,
  66     return_pc_is_thread_saved_exception_pc
  67   };
  68 
  69   static OopMap* push_frame_reg_args_and_save_live_registers(MacroAssembler* masm,
  70                          int* out_frame_size_in_bytes,
  71                          bool generate_oop_map,
  72                          int return_pc_adjustment,
  73                          ReturnPCLocation return_pc_location);
  74   static void    restore_live_registers_and_pop_frame(MacroAssembler* masm,
  75                          int frame_size_in_bytes,
  76                          bool restore_ctr);
  77 
  78   static void push_frame_and_save_argument_registers(MacroAssembler* masm,
  79                          Register r_temp,
  80                          int frame_size,
  81                          int total_args,
  82                          const VMRegPair *regs, const VMRegPair *regs2 = NULL);
  83   static void restore_argument_registers_and_pop_frame(MacroAssembler*masm,
  84                          int frame_size,
  85                          int total_args,
  86                          const VMRegPair *regs, const VMRegPair *regs2 = NULL);
  87 
  88   // During deoptimization only the result registers need to be restored
  89   // all the other values have already been extracted.
  90   static void restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes);
  91 
  92   // Constants and data structures:
  93 
  94   typedef enum {
  95     int_reg           = 0,
  96     float_reg         = 1,
  97     special_reg       = 2
  98   } RegisterType;
  99 
 100   typedef enum {
 101     reg_size          = 8,
 102     half_reg_size     = reg_size / 2,
 103   } RegisterConstants;
 104 
 105   typedef struct {
 106     RegisterType        reg_type;
 107     int                 reg_num;
 108     VMReg               vmreg;
 109   } LiveRegType;
 110 };
 111 
 112 
 113 #define RegisterSaver_LiveSpecialReg(regname) \
 114   { RegisterSaver::special_reg, regname->encoding(), regname->as_VMReg() }
 115 
 116 #define RegisterSaver_LiveIntReg(regname) \
 117   { RegisterSaver::int_reg,     regname->encoding(), regname->as_VMReg() }
 118 
 119 #define RegisterSaver_LiveFloatReg(regname) \
 120   { RegisterSaver::float_reg,   regname->encoding(), regname->as_VMReg() }
 121 
 122 static const RegisterSaver::LiveRegType RegisterSaver_LiveRegs[] = {
 123   // Live registers which get spilled to the stack. Register
 124   // positions in this array correspond directly to the stack layout.
 125 
 126   //
 127   // live special registers:
 128   //
 129   RegisterSaver_LiveSpecialReg(SR_CTR),
 130   //
 131   // live float registers:
 132   //
 133   RegisterSaver_LiveFloatReg( F0  ),
 134   RegisterSaver_LiveFloatReg( F1  ),
 135   RegisterSaver_LiveFloatReg( F2  ),
 136   RegisterSaver_LiveFloatReg( F3  ),
 137   RegisterSaver_LiveFloatReg( F4  ),
 138   RegisterSaver_LiveFloatReg( F5  ),
 139   RegisterSaver_LiveFloatReg( F6  ),
 140   RegisterSaver_LiveFloatReg( F7  ),
 141   RegisterSaver_LiveFloatReg( F8  ),
 142   RegisterSaver_LiveFloatReg( F9  ),
 143   RegisterSaver_LiveFloatReg( F10 ),
 144   RegisterSaver_LiveFloatReg( F11 ),
 145   RegisterSaver_LiveFloatReg( F12 ),
 146   RegisterSaver_LiveFloatReg( F13 ),
 147   RegisterSaver_LiveFloatReg( F14 ),
 148   RegisterSaver_LiveFloatReg( F15 ),
 149   RegisterSaver_LiveFloatReg( F16 ),
 150   RegisterSaver_LiveFloatReg( F17 ),
 151   RegisterSaver_LiveFloatReg( F18 ),
 152   RegisterSaver_LiveFloatReg( F19 ),
 153   RegisterSaver_LiveFloatReg( F20 ),
 154   RegisterSaver_LiveFloatReg( F21 ),
 155   RegisterSaver_LiveFloatReg( F22 ),
 156   RegisterSaver_LiveFloatReg( F23 ),
 157   RegisterSaver_LiveFloatReg( F24 ),
 158   RegisterSaver_LiveFloatReg( F25 ),
 159   RegisterSaver_LiveFloatReg( F26 ),
 160   RegisterSaver_LiveFloatReg( F27 ),
 161   RegisterSaver_LiveFloatReg( F28 ),
 162   RegisterSaver_LiveFloatReg( F29 ),
 163   RegisterSaver_LiveFloatReg( F30 ),
 164   RegisterSaver_LiveFloatReg( F31 ),
 165   //
 166   // live integer registers:
 167   //
 168   RegisterSaver_LiveIntReg(   R0  ),
 169   //RegisterSaver_LiveIntReg( R1  ), // stack pointer
 170   RegisterSaver_LiveIntReg(   R2  ),
 171   RegisterSaver_LiveIntReg(   R3  ),
 172   RegisterSaver_LiveIntReg(   R4  ),
 173   RegisterSaver_LiveIntReg(   R5  ),
 174   RegisterSaver_LiveIntReg(   R6  ),
 175   RegisterSaver_LiveIntReg(   R7  ),
 176   RegisterSaver_LiveIntReg(   R8  ),
 177   RegisterSaver_LiveIntReg(   R9  ),
 178   RegisterSaver_LiveIntReg(   R10 ),
 179   RegisterSaver_LiveIntReg(   R11 ),
 180   RegisterSaver_LiveIntReg(   R12 ),
 181   //RegisterSaver_LiveIntReg( R13 ), // system thread id
 182   RegisterSaver_LiveIntReg(   R14 ),
 183   RegisterSaver_LiveIntReg(   R15 ),
 184   RegisterSaver_LiveIntReg(   R16 ),
 185   RegisterSaver_LiveIntReg(   R17 ),
 186   RegisterSaver_LiveIntReg(   R18 ),
 187   RegisterSaver_LiveIntReg(   R19 ),
 188   RegisterSaver_LiveIntReg(   R20 ),
 189   RegisterSaver_LiveIntReg(   R21 ),
 190   RegisterSaver_LiveIntReg(   R22 ),
 191   RegisterSaver_LiveIntReg(   R23 ),
 192   RegisterSaver_LiveIntReg(   R24 ),
 193   RegisterSaver_LiveIntReg(   R25 ),
 194   RegisterSaver_LiveIntReg(   R26 ),
 195   RegisterSaver_LiveIntReg(   R27 ),
 196   RegisterSaver_LiveIntReg(   R28 ),
 197   RegisterSaver_LiveIntReg(   R29 ),
 198   RegisterSaver_LiveIntReg(   R30 ),
 199   RegisterSaver_LiveIntReg(   R31 ), // must be the last register (see save/restore functions below)
 200 };
 201 
 202 OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssembler* masm,
 203                          int* out_frame_size_in_bytes,
 204                          bool generate_oop_map,
 205                          int return_pc_adjustment,
 206                          ReturnPCLocation return_pc_location) {
 207   // Push an abi_reg_args-frame and store all registers which may be live.
 208   // If requested, create an OopMap: Record volatile registers as
 209   // callee-save values in an OopMap so their save locations will be
 210   // propagated to the RegisterMap of the caller frame during
 211   // StackFrameStream construction (needed for deoptimization; see
 212   // compiledVFrame::create_stack_value).
 213   // If return_pc_adjustment != 0 adjust the return pc by return_pc_adjustment.
 214 
 215   int i;
 216   int offset;
 217 
 218   // calcualte frame size
 219   const int regstosave_num       = sizeof(RegisterSaver_LiveRegs) /
 220                                    sizeof(RegisterSaver::LiveRegType);
 221   const int register_save_size   = regstosave_num * reg_size;
 222   const int frame_size_in_bytes  = round_to(register_save_size, frame::alignment_in_bytes)
 223                                    + frame::abi_reg_args_size;
 224   *out_frame_size_in_bytes       = frame_size_in_bytes;
 225   const int frame_size_in_slots  = frame_size_in_bytes / sizeof(jint);
 226   const int register_save_offset = frame_size_in_bytes - register_save_size;
 227 
 228   // OopMap frame size is in c2 stack slots (sizeof(jint)) not bytes or words.
 229   OopMap* map = generate_oop_map ? new OopMap(frame_size_in_slots, 0) : NULL;
 230 
 231   BLOCK_COMMENT("push_frame_reg_args_and_save_live_registers {");
 232 
 233   // Save r31 in the last slot of the not yet pushed frame so that we
 234   // can use it as scratch reg.
 235   __ std(R31, -reg_size, R1_SP);
 236   assert(-reg_size == register_save_offset - frame_size_in_bytes + ((regstosave_num-1)*reg_size),
 237          "consistency check");
 238 
 239   // save the flags
 240   // Do the save_LR_CR by hand and adjust the return pc if requested.
 241   __ mfcr(R31);
 242   __ std(R31, _abi(cr), R1_SP);
 243   switch (return_pc_location) {
 244     case return_pc_is_lr:    __ mflr(R31);           break;
 245     case return_pc_is_r4:    __ mr(R31, R4);     break;
 246     case return_pc_is_thread_saved_exception_pc:
 247                              __ ld(R31, thread_(saved_exception_pc)); break;
 248     default: ShouldNotReachHere();
 249   }
 250   if (return_pc_adjustment != 0) {
 251     __ addi(R31, R31, return_pc_adjustment);
 252   }
 253   __ std(R31, _abi(lr), R1_SP);
 254 
 255   // push a new frame
 256   __ push_frame(frame_size_in_bytes, R31);
 257 
 258   // save all registers (ints and floats)
 259   offset = register_save_offset;
 260   for (int i = 0; i < regstosave_num; i++) {
 261     int reg_num  = RegisterSaver_LiveRegs[i].reg_num;
 262     int reg_type = RegisterSaver_LiveRegs[i].reg_type;
 263 
 264     switch (reg_type) {
 265       case RegisterSaver::int_reg: {
 266         if (reg_num != 31) { // We spilled R31 right at the beginning.
 267           __ std(as_Register(reg_num), offset, R1_SP);
 268         }
 269         break;
 270       }
 271       case RegisterSaver::float_reg: {
 272         __ stfd(as_FloatRegister(reg_num), offset, R1_SP);
 273         break;
 274       }
 275       case RegisterSaver::special_reg: {
 276         if (reg_num == SR_CTR_SpecialRegisterEnumValue) {
 277           __ mfctr(R31);
 278           __ std(R31, offset, R1_SP);
 279         } else {
 280           Unimplemented();
 281         }
 282         break;
 283       }
 284       default:
 285         ShouldNotReachHere();
 286     }
 287 
 288     if (generate_oop_map) {
 289       map->set_callee_saved(VMRegImpl::stack2reg(offset>>2),
 290                             RegisterSaver_LiveRegs[i].vmreg);
 291       map->set_callee_saved(VMRegImpl::stack2reg((offset + half_reg_size)>>2),
 292                             RegisterSaver_LiveRegs[i].vmreg->next());
 293     }
 294     offset += reg_size;
 295   }
 296 
 297   BLOCK_COMMENT("} push_frame_reg_args_and_save_live_registers");
 298 
 299   // And we're done.
 300   return map;
 301 }
 302 
 303 
 304 // Pop the current frame and restore all the registers that we
 305 // saved.
 306 void RegisterSaver::restore_live_registers_and_pop_frame(MacroAssembler* masm,
 307                                                          int frame_size_in_bytes,
 308                                                          bool restore_ctr) {
 309   int i;
 310   int offset;
 311   const int regstosave_num       = sizeof(RegisterSaver_LiveRegs) /
 312                                    sizeof(RegisterSaver::LiveRegType);
 313   const int register_save_size   = regstosave_num * reg_size;
 314   const int register_save_offset = frame_size_in_bytes - register_save_size;
 315 
 316   BLOCK_COMMENT("restore_live_registers_and_pop_frame {");
 317 
 318   // restore all registers (ints and floats)
 319   offset = register_save_offset;
 320   for (int i = 0; i < regstosave_num; i++) {
 321     int reg_num  = RegisterSaver_LiveRegs[i].reg_num;
 322     int reg_type = RegisterSaver_LiveRegs[i].reg_type;
 323 
 324     switch (reg_type) {
 325       case RegisterSaver::int_reg: {
 326         if (reg_num != 31) // R31 restored at the end, it's the tmp reg!
 327           __ ld(as_Register(reg_num), offset, R1_SP);
 328         break;
 329       }
 330       case RegisterSaver::float_reg: {
 331         __ lfd(as_FloatRegister(reg_num), offset, R1_SP);
 332         break;
 333       }
 334       case RegisterSaver::special_reg: {
 335         if (reg_num == SR_CTR_SpecialRegisterEnumValue) {
 336           if (restore_ctr) { // Nothing to do here if ctr already contains the next address.
 337             __ ld(R31, offset, R1_SP);
 338             __ mtctr(R31);
 339           }
 340         } else {
 341           Unimplemented();
 342         }
 343         break;
 344       }
 345       default:
 346         ShouldNotReachHere();
 347     }
 348     offset += reg_size;
 349   }
 350 
 351   // pop the frame
 352   __ pop_frame();
 353 
 354   // restore the flags
 355   __ restore_LR_CR(R31);
 356 
 357   // restore scratch register's value
 358   __ ld(R31, -reg_size, R1_SP);
 359 
 360   BLOCK_COMMENT("} restore_live_registers_and_pop_frame");
 361 }
 362 
 363 void RegisterSaver::push_frame_and_save_argument_registers(MacroAssembler* masm, Register r_temp,
 364                                                            int frame_size,int total_args, const VMRegPair *regs,
 365                                                            const VMRegPair *regs2) {
 366   __ push_frame(frame_size, r_temp);
 367   int st_off = frame_size - wordSize;
 368   for (int i = 0; i < total_args; i++) {
 369     VMReg r_1 = regs[i].first();
 370     VMReg r_2 = regs[i].second();
 371     if (!r_1->is_valid()) {
 372       assert(!r_2->is_valid(), "");
 373       continue;
 374     }
 375     if (r_1->is_Register()) {
 376       Register r = r_1->as_Register();
 377       __ std(r, st_off, R1_SP);
 378       st_off -= wordSize;
 379     } else if (r_1->is_FloatRegister()) {
 380       FloatRegister f = r_1->as_FloatRegister();
 381       __ stfd(f, st_off, R1_SP);
 382       st_off -= wordSize;
 383     }
 384   }
 385   if (regs2 != NULL) {
 386     for (int i = 0; i < total_args; i++) {
 387       VMReg r_1 = regs2[i].first();
 388       VMReg r_2 = regs2[i].second();
 389       if (!r_1->is_valid()) {
 390         assert(!r_2->is_valid(), "");
 391         continue;
 392       }
 393       if (r_1->is_Register()) {
 394         Register r = r_1->as_Register();
 395         __ std(r, st_off, R1_SP);
 396         st_off -= wordSize;
 397       } else if (r_1->is_FloatRegister()) {
 398         FloatRegister f = r_1->as_FloatRegister();
 399         __ stfd(f, st_off, R1_SP);
 400         st_off -= wordSize;
 401       }
 402     }
 403   }
 404 }
 405 
 406 void RegisterSaver::restore_argument_registers_and_pop_frame(MacroAssembler*masm, int frame_size,
 407                                                              int total_args, const VMRegPair *regs,
 408                                                              const VMRegPair *regs2) {
 409   int st_off = frame_size - wordSize;
 410   for (int i = 0; i < total_args; i++) {
 411     VMReg r_1 = regs[i].first();
 412     VMReg r_2 = regs[i].second();
 413     if (r_1->is_Register()) {
 414       Register r = r_1->as_Register();
 415       __ ld(r, st_off, R1_SP);
 416       st_off -= wordSize;
 417     } else if (r_1->is_FloatRegister()) {
 418       FloatRegister f = r_1->as_FloatRegister();
 419       __ lfd(f, st_off, R1_SP);
 420       st_off -= wordSize;
 421     }
 422   }
 423   if (regs2 != NULL)
 424     for (int i = 0; i < total_args; i++) {
 425       VMReg r_1 = regs2[i].first();
 426       VMReg r_2 = regs2[i].second();
 427       if (r_1->is_Register()) {
 428         Register r = r_1->as_Register();
 429         __ ld(r, st_off, R1_SP);
 430         st_off -= wordSize;
 431       } else if (r_1->is_FloatRegister()) {
 432         FloatRegister f = r_1->as_FloatRegister();
 433         __ lfd(f, st_off, R1_SP);
 434         st_off -= wordSize;
 435       }
 436     }
 437   __ pop_frame();
 438 }
 439 
 440 // Restore the registers that might be holding a result.
 441 void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes) {
 442   int i;
 443   int offset;
 444   const int regstosave_num       = sizeof(RegisterSaver_LiveRegs) /
 445                                    sizeof(RegisterSaver::LiveRegType);
 446   const int register_save_size   = regstosave_num * reg_size;
 447   const int register_save_offset = frame_size_in_bytes - register_save_size;
 448 
 449   // restore all result registers (ints and floats)
 450   offset = register_save_offset;
 451   for (int i = 0; i < regstosave_num; i++) {
 452     int reg_num  = RegisterSaver_LiveRegs[i].reg_num;
 453     int reg_type = RegisterSaver_LiveRegs[i].reg_type;
 454     switch (reg_type) {
 455       case RegisterSaver::int_reg: {
 456         if (as_Register(reg_num)==R3_RET) // int result_reg
 457           __ ld(as_Register(reg_num), offset, R1_SP);
 458         break;
 459       }
 460       case RegisterSaver::float_reg: {
 461         if (as_FloatRegister(reg_num)==F1_RET) // float result_reg
 462           __ lfd(as_FloatRegister(reg_num), offset, R1_SP);
 463         break;
 464       }
 465       case RegisterSaver::special_reg: {
 466         // Special registers don't hold a result.
 467         break;
 468       }
 469       default:
 470         ShouldNotReachHere();
 471     }
 472     offset += reg_size;
 473   }
 474 }
 475 
 476 // Is vector's size (in bytes) bigger than a size saved by default?
 477 bool SharedRuntime::is_wide_vector(int size) {
 478   // Note, MaxVectorSize == 8 on PPC64.
 479   assert(size <= 8, "%d bytes vectors are not supported", size);
 480   return size > 8;
 481 }
 482 #ifdef COMPILER2
 483 static int reg2slot(VMReg r) {
 484   return r->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 485 }
 486 
 487 static int reg2offset(VMReg r) {
 488   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 489 }
 490 #endif
 491 
 492 // ---------------------------------------------------------------------------
 493 // Read the array of BasicTypes from a signature, and compute where the
 494 // arguments should go. Values in the VMRegPair regs array refer to 4-byte
 495 // quantities. Values less than VMRegImpl::stack0 are registers, those above
 496 // refer to 4-byte stack slots. All stack slots are based off of the stack pointer
 497 // as framesizes are fixed.
 498 // VMRegImpl::stack0 refers to the first slot 0(sp).
 499 // and VMRegImpl::stack0+1 refers to the memory word 4-bytes higher. Register
 500 // up to RegisterImpl::number_of_registers) are the 64-bit
 501 // integer registers.
 502 
 503 // Note: the INPUTS in sig_bt are in units of Java argument words, which are
 504 // either 32-bit or 64-bit depending on the build. The OUTPUTS are in 32-bit
 505 // units regardless of build. Of course for i486 there is no 64 bit build
 506 
 507 // The Java calling convention is a "shifted" version of the C ABI.
 508 // By skipping the first C ABI register we can call non-static jni methods
 509 // with small numbers of arguments without having to shuffle the arguments
 510 // at all. Since we control the java ABI we ought to at least get some
 511 // advantage out of it.
 512 
 513 const VMReg java_iarg_reg[8] = {
 514   R3->as_VMReg(),
 515   R4->as_VMReg(),
 516   R5->as_VMReg(),
 517   R6->as_VMReg(),
 518   R7->as_VMReg(),
 519   R8->as_VMReg(),
 520   R9->as_VMReg(),
 521   R10->as_VMReg()
 522 };
 523 
 524 const VMReg java_farg_reg[13] = {
 525   F1->as_VMReg(),
 526   F2->as_VMReg(),
 527   F3->as_VMReg(),
 528   F4->as_VMReg(),
 529   F5->as_VMReg(),
 530   F6->as_VMReg(),
 531   F7->as_VMReg(),
 532   F8->as_VMReg(),
 533   F9->as_VMReg(),
 534   F10->as_VMReg(),
 535   F11->as_VMReg(),
 536   F12->as_VMReg(),
 537   F13->as_VMReg()
 538 };
 539 
 540 const int num_java_iarg_registers = sizeof(java_iarg_reg) / sizeof(java_iarg_reg[0]);
 541 const int num_java_farg_registers = sizeof(java_farg_reg) / sizeof(java_farg_reg[0]);
 542 
 543 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
 544                                            VMRegPair *regs,
 545                                            int total_args_passed,
 546                                            int is_outgoing) {
 547   // C2c calling conventions for compiled-compiled calls.
 548   // Put 8 ints/longs into registers _AND_ 13 float/doubles into
 549   // registers _AND_ put the rest on the stack.
 550 
 551   const int inc_stk_for_intfloat   = 1; // 1 slots for ints and floats
 552   const int inc_stk_for_longdouble = 2; // 2 slots for longs and doubles
 553 
 554   int i;
 555   VMReg reg;
 556   int stk = 0;
 557   int ireg = 0;
 558   int freg = 0;
 559 
 560   // We put the first 8 arguments into registers and the rest on the
 561   // stack, float arguments are already in their argument registers
 562   // due to c2c calling conventions (see calling_convention).
 563   for (int i = 0; i < total_args_passed; ++i) {
 564     switch(sig_bt[i]) {
 565     case T_BOOLEAN:
 566     case T_CHAR:
 567     case T_BYTE:
 568     case T_SHORT:
 569     case T_INT:
 570       if (ireg < num_java_iarg_registers) {
 571         // Put int/ptr in register
 572         reg = java_iarg_reg[ireg];
 573         ++ireg;
 574       } else {
 575         // Put int/ptr on stack.
 576         reg = VMRegImpl::stack2reg(stk);
 577         stk += inc_stk_for_intfloat;
 578       }
 579       regs[i].set1(reg);
 580       break;
 581     case T_LONG:
 582       assert(sig_bt[i+1] == T_VOID, "expecting half");
 583       if (ireg < num_java_iarg_registers) {
 584         // Put long in register.
 585         reg = java_iarg_reg[ireg];
 586         ++ireg;
 587       } else {
 588         // Put long on stack. They must be aligned to 2 slots.
 589         if (stk & 0x1) ++stk;
 590         reg = VMRegImpl::stack2reg(stk);
 591         stk += inc_stk_for_longdouble;
 592       }
 593       regs[i].set2(reg);
 594       break;
 595     case T_OBJECT:
 596     case T_ARRAY:
 597     case T_ADDRESS:
 598       if (ireg < num_java_iarg_registers) {
 599         // Put ptr in register.
 600         reg = java_iarg_reg[ireg];
 601         ++ireg;
 602       } else {
 603         // Put ptr on stack. Objects must be aligned to 2 slots too,
 604         // because "64-bit pointers record oop-ishness on 2 aligned
 605         // adjacent registers." (see OopFlow::build_oop_map).
 606         if (stk & 0x1) ++stk;
 607         reg = VMRegImpl::stack2reg(stk);
 608         stk += inc_stk_for_longdouble;
 609       }
 610       regs[i].set2(reg);
 611       break;
 612     case T_FLOAT:
 613       if (freg < num_java_farg_registers) {
 614         // Put float in register.
 615         reg = java_farg_reg[freg];
 616         ++freg;
 617       } else {
 618         // Put float on stack.
 619         reg = VMRegImpl::stack2reg(stk);
 620         stk += inc_stk_for_intfloat;
 621       }
 622       regs[i].set1(reg);
 623       break;
 624     case T_DOUBLE:
 625       assert(sig_bt[i+1] == T_VOID, "expecting half");
 626       if (freg < num_java_farg_registers) {
 627         // Put double in register.
 628         reg = java_farg_reg[freg];
 629         ++freg;
 630       } else {
 631         // Put double on stack. They must be aligned to 2 slots.
 632         if (stk & 0x1) ++stk;
 633         reg = VMRegImpl::stack2reg(stk);
 634         stk += inc_stk_for_longdouble;
 635       }
 636       regs[i].set2(reg);
 637       break;
 638     case T_VOID:
 639       // Do not count halves.
 640       regs[i].set_bad();
 641       break;
 642     default:
 643       ShouldNotReachHere();
 644     }
 645   }
 646   return round_to(stk, 2);
 647 }
 648 
 649 #ifdef COMPILER2
 650 // Calling convention for calling C code.
 651 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
 652                                         VMRegPair *regs,
 653                                         VMRegPair *regs2,
 654                                         int total_args_passed) {
 655   // Calling conventions for C runtime calls and calls to JNI native methods.
 656   //
 657   // PPC64 convention: Hoist the first 8 int/ptr/long's in the first 8
 658   // int regs, leaving int regs undefined if the arg is flt/dbl. Hoist
 659   // the first 13 flt/dbl's in the first 13 fp regs but additionally
 660   // copy flt/dbl to the stack if they are beyond the 8th argument.
 661 
 662   const VMReg iarg_reg[8] = {
 663     R3->as_VMReg(),
 664     R4->as_VMReg(),
 665     R5->as_VMReg(),
 666     R6->as_VMReg(),
 667     R7->as_VMReg(),
 668     R8->as_VMReg(),
 669     R9->as_VMReg(),
 670     R10->as_VMReg()
 671   };
 672 
 673   const VMReg farg_reg[13] = {
 674     F1->as_VMReg(),
 675     F2->as_VMReg(),
 676     F3->as_VMReg(),
 677     F4->as_VMReg(),
 678     F5->as_VMReg(),
 679     F6->as_VMReg(),
 680     F7->as_VMReg(),
 681     F8->as_VMReg(),
 682     F9->as_VMReg(),
 683     F10->as_VMReg(),
 684     F11->as_VMReg(),
 685     F12->as_VMReg(),
 686     F13->as_VMReg()
 687   };
 688 
 689   // Check calling conventions consistency.
 690   assert(sizeof(iarg_reg) / sizeof(iarg_reg[0]) == Argument::n_int_register_parameters_c &&
 691          sizeof(farg_reg) / sizeof(farg_reg[0]) == Argument::n_float_register_parameters_c,
 692          "consistency");
 693 
 694   // `Stk' counts stack slots. Due to alignment, 32 bit values occupy
 695   // 2 such slots, like 64 bit values do.
 696   const int inc_stk_for_intfloat   = 2; // 2 slots for ints and floats
 697   const int inc_stk_for_longdouble = 2; // 2 slots for longs and doubles
 698 
 699   int i;
 700   VMReg reg;
 701   // Leave room for C-compatible ABI_REG_ARGS.
 702   int stk = (frame::abi_reg_args_size - frame::jit_out_preserve_size) / VMRegImpl::stack_slot_size;
 703   int arg = 0;
 704   int freg = 0;
 705 
 706   // Avoid passing C arguments in the wrong stack slots.
 707 #if defined(ABI_ELFv2)
 708   assert((SharedRuntime::out_preserve_stack_slots() + stk) * VMRegImpl::stack_slot_size == 96,
 709          "passing C arguments in wrong stack slots");
 710 #else
 711   assert((SharedRuntime::out_preserve_stack_slots() + stk) * VMRegImpl::stack_slot_size == 112,
 712          "passing C arguments in wrong stack slots");
 713 #endif
 714   // We fill-out regs AND regs2 if an argument must be passed in a
 715   // register AND in a stack slot. If regs2 is NULL in such a
 716   // situation, we bail-out with a fatal error.
 717   for (int i = 0; i < total_args_passed; ++i, ++arg) {
 718     // Initialize regs2 to BAD.
 719     if (regs2 != NULL) regs2[i].set_bad();
 720 
 721     switch(sig_bt[i]) {
 722 
 723     //
 724     // If arguments 0-7 are integers, they are passed in integer registers.
 725     // Argument i is placed in iarg_reg[i].
 726     //
 727     case T_BOOLEAN:
 728     case T_CHAR:
 729     case T_BYTE:
 730     case T_SHORT:
 731     case T_INT:
 732       // We must cast ints to longs and use full 64 bit stack slots
 733       // here.  Thus fall through, handle as long.
 734     case T_LONG:
 735     case T_OBJECT:
 736     case T_ARRAY:
 737     case T_ADDRESS:
 738     case T_METADATA:
 739       // Oops are already boxed if required (JNI).
 740       if (arg < Argument::n_int_register_parameters_c) {
 741         reg = iarg_reg[arg];
 742       } else {
 743         reg = VMRegImpl::stack2reg(stk);
 744         stk += inc_stk_for_longdouble;
 745       }
 746       regs[i].set2(reg);
 747       break;
 748 
 749     //
 750     // Floats are treated differently from int regs:  The first 13 float arguments
 751     // are passed in registers (not the float args among the first 13 args).
 752     // Thus argument i is NOT passed in farg_reg[i] if it is float.  It is passed
 753     // in farg_reg[j] if argument i is the j-th float argument of this call.
 754     //
 755     case T_FLOAT:
 756 #if defined(LINUX)
 757       // Linux uses ELF ABI. Both original ELF and ELFv2 ABIs have float
 758       // in the least significant word of an argument slot.
 759 #if defined(VM_LITTLE_ENDIAN)
 760 #define FLOAT_WORD_OFFSET_IN_SLOT 0
 761 #else
 762 #define FLOAT_WORD_OFFSET_IN_SLOT 1
 763 #endif
 764 #elif defined(AIX)
 765       // Although AIX runs on big endian CPU, float is in the most
 766       // significant word of an argument slot.
 767 #define FLOAT_WORD_OFFSET_IN_SLOT 0
 768 #else
 769 #error "unknown OS"
 770 #endif
 771       if (freg < Argument::n_float_register_parameters_c) {
 772         // Put float in register ...
 773         reg = farg_reg[freg];
 774         ++freg;
 775 
 776         // Argument i for i > 8 is placed on the stack even if it's
 777         // placed in a register (if it's a float arg). Aix disassembly
 778         // shows that xlC places these float args on the stack AND in
 779         // a register. This is not documented, but we follow this
 780         // convention, too.
 781         if (arg >= Argument::n_regs_not_on_stack_c) {
 782           // ... and on the stack.
 783           guarantee(regs2 != NULL, "must pass float in register and stack slot");
 784           VMReg reg2 = VMRegImpl::stack2reg(stk + FLOAT_WORD_OFFSET_IN_SLOT);
 785           regs2[i].set1(reg2);
 786           stk += inc_stk_for_intfloat;
 787         }
 788 
 789       } else {
 790         // Put float on stack.
 791         reg = VMRegImpl::stack2reg(stk + FLOAT_WORD_OFFSET_IN_SLOT);
 792         stk += inc_stk_for_intfloat;
 793       }
 794       regs[i].set1(reg);
 795       break;
 796     case T_DOUBLE:
 797       assert(sig_bt[i+1] == T_VOID, "expecting half");
 798       if (freg < Argument::n_float_register_parameters_c) {
 799         // Put double in register ...
 800         reg = farg_reg[freg];
 801         ++freg;
 802 
 803         // Argument i for i > 8 is placed on the stack even if it's
 804         // placed in a register (if it's a double arg). Aix disassembly
 805         // shows that xlC places these float args on the stack AND in
 806         // a register. This is not documented, but we follow this
 807         // convention, too.
 808         if (arg >= Argument::n_regs_not_on_stack_c) {
 809           // ... and on the stack.
 810           guarantee(regs2 != NULL, "must pass float in register and stack slot");
 811           VMReg reg2 = VMRegImpl::stack2reg(stk);
 812           regs2[i].set2(reg2);
 813           stk += inc_stk_for_longdouble;
 814         }
 815       } else {
 816         // Put double on stack.
 817         reg = VMRegImpl::stack2reg(stk);
 818         stk += inc_stk_for_longdouble;
 819       }
 820       regs[i].set2(reg);
 821       break;
 822 
 823     case T_VOID:
 824       // Do not count halves.
 825       regs[i].set_bad();
 826       --arg;
 827       break;
 828     default:
 829       ShouldNotReachHere();
 830     }
 831   }
 832 
 833   return round_to(stk, 2);
 834 }
 835 #endif // COMPILER2
 836 
 837 static address gen_c2i_adapter(MacroAssembler *masm,
 838                             int total_args_passed,
 839                             int comp_args_on_stack,
 840                             const BasicType *sig_bt,
 841                             const VMRegPair *regs,
 842                             Label& call_interpreter,
 843                             const Register& ientry) {
 844 
 845   address c2i_entrypoint;
 846 
 847   const Register sender_SP = R21_sender_SP; // == R21_tmp1
 848   const Register code      = R22_tmp2;
 849   //const Register ientry  = R23_tmp3;
 850   const Register value_regs[] = { R24_tmp4, R25_tmp5, R26_tmp6 };
 851   const int num_value_regs = sizeof(value_regs) / sizeof(Register);
 852   int value_regs_index = 0;
 853 
 854   const Register return_pc = R27_tmp7;
 855   const Register tmp       = R28_tmp8;
 856 
 857   assert_different_registers(sender_SP, code, ientry, return_pc, tmp);
 858 
 859   // Adapter needs TOP_IJAVA_FRAME_ABI.
 860   const int adapter_size = frame::top_ijava_frame_abi_size +
 861                            round_to(total_args_passed * wordSize, frame::alignment_in_bytes);
 862 
 863   // regular (verified) c2i entry point
 864   c2i_entrypoint = __ pc();
 865 
 866   // Does compiled code exists? If yes, patch the caller's callsite.
 867   __ ld(code, method_(code));
 868   __ cmpdi(CCR0, code, 0);
 869   __ ld(ientry, method_(interpreter_entry)); // preloaded
 870   __ beq(CCR0, call_interpreter);
 871 
 872 
 873   // Patch caller's callsite, method_(code) was not NULL which means that
 874   // compiled code exists.
 875   __ mflr(return_pc);
 876   __ std(return_pc, _abi(lr), R1_SP);
 877   RegisterSaver::push_frame_and_save_argument_registers(masm, tmp, adapter_size, total_args_passed, regs);
 878 
 879   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite), R19_method, return_pc);
 880 
 881   RegisterSaver::restore_argument_registers_and_pop_frame(masm, adapter_size, total_args_passed, regs);
 882   __ ld(return_pc, _abi(lr), R1_SP);
 883   __ ld(ientry, method_(interpreter_entry)); // preloaded
 884   __ mtlr(return_pc);
 885 
 886 
 887   // Call the interpreter.
 888   __ BIND(call_interpreter);
 889   __ mtctr(ientry);
 890 
 891   // Get a copy of the current SP for loading caller's arguments.
 892   __ mr(sender_SP, R1_SP);
 893 
 894   // Add space for the adapter.
 895   __ resize_frame(-adapter_size, R12_scratch2);
 896 
 897   int st_off = adapter_size - wordSize;
 898 
 899   // Write the args into the outgoing interpreter space.
 900   for (int i = 0; i < total_args_passed; i++) {
 901     VMReg r_1 = regs[i].first();
 902     VMReg r_2 = regs[i].second();
 903     if (!r_1->is_valid()) {
 904       assert(!r_2->is_valid(), "");
 905       continue;
 906     }
 907     if (r_1->is_stack()) {
 908       Register tmp_reg = value_regs[value_regs_index];
 909       value_regs_index = (value_regs_index + 1) % num_value_regs;
 910       // The calling convention produces OptoRegs that ignore the out
 911       // preserve area (JIT's ABI). We must account for it here.
 912       int ld_off = (r_1->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 913       if (!r_2->is_valid()) {
 914         __ lwz(tmp_reg, ld_off, sender_SP);
 915       } else {
 916         __ ld(tmp_reg, ld_off, sender_SP);
 917       }
 918       // Pretend stack targets were loaded into tmp_reg.
 919       r_1 = tmp_reg->as_VMReg();
 920     }
 921 
 922     if (r_1->is_Register()) {
 923       Register r = r_1->as_Register();
 924       if (!r_2->is_valid()) {
 925         __ stw(r, st_off, R1_SP);
 926         st_off-=wordSize;
 927       } else {
 928         // Longs are given 2 64-bit slots in the interpreter, but the
 929         // data is passed in only 1 slot.
 930         if (sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 931           DEBUG_ONLY( __ li(tmp, 0); __ std(tmp, st_off, R1_SP); )
 932           st_off-=wordSize;
 933         }
 934         __ std(r, st_off, R1_SP);
 935         st_off-=wordSize;
 936       }
 937     } else {
 938       assert(r_1->is_FloatRegister(), "");
 939       FloatRegister f = r_1->as_FloatRegister();
 940       if (!r_2->is_valid()) {
 941         __ stfs(f, st_off, R1_SP);
 942         st_off-=wordSize;
 943       } else {
 944         // In 64bit, doubles are given 2 64-bit slots in the interpreter, but the
 945         // data is passed in only 1 slot.
 946         // One of these should get known junk...
 947         DEBUG_ONLY( __ li(tmp, 0); __ std(tmp, st_off, R1_SP); )
 948         st_off-=wordSize;
 949         __ stfd(f, st_off, R1_SP);
 950         st_off-=wordSize;
 951       }
 952     }
 953   }
 954 
 955   // Jump to the interpreter just as if interpreter was doing it.
 956 
 957 #ifdef CC_INTERP
 958   const Register tos = R17_tos;
 959 #else
 960   const Register tos = R15_esp;
 961   __ load_const_optimized(R25_templateTableBase, (address)Interpreter::dispatch_table((TosState)0), R11_scratch1);
 962 #endif
 963 
 964   // load TOS
 965   __ addi(tos, R1_SP, st_off);
 966 
 967   // Frame_manager expects initial_caller_sp (= SP without resize by c2i) in R21_tmp1.
 968   assert(sender_SP == R21_sender_SP, "passing initial caller's SP in wrong register");
 969   __ bctr();
 970 
 971   return c2i_entrypoint;
 972 }
 973 
 974 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 975                                     int total_args_passed,
 976                                     int comp_args_on_stack,
 977                                     const BasicType *sig_bt,
 978                                     const VMRegPair *regs) {
 979 
 980   // Load method's entry-point from method.
 981   __ ld(R12_scratch2, in_bytes(Method::from_compiled_offset()), R19_method);
 982   __ mtctr(R12_scratch2);
 983 
 984   // We will only enter here from an interpreted frame and never from after
 985   // passing thru a c2i. Azul allowed this but we do not. If we lose the
 986   // race and use a c2i we will remain interpreted for the race loser(s).
 987   // This removes all sorts of headaches on the x86 side and also eliminates
 988   // the possibility of having c2i -> i2c -> c2i -> ... endless transitions.
 989 
 990   // Note: r13 contains the senderSP on entry. We must preserve it since
 991   // we may do a i2c -> c2i transition if we lose a race where compiled
 992   // code goes non-entrant while we get args ready.
 993   // In addition we use r13 to locate all the interpreter args as
 994   // we must align the stack to 16 bytes on an i2c entry else we
 995   // lose alignment we expect in all compiled code and register
 996   // save code can segv when fxsave instructions find improperly
 997   // aligned stack pointer.
 998 
 999 #ifdef CC_INTERP
1000   const Register ld_ptr = R17_tos;
1001 #else
1002   const Register ld_ptr = R15_esp;
1003 #endif
1004 
1005   const Register value_regs[] = { R22_tmp2, R23_tmp3, R24_tmp4, R25_tmp5, R26_tmp6 };
1006   const int num_value_regs = sizeof(value_regs) / sizeof(Register);
1007   int value_regs_index = 0;
1008 
1009   int ld_offset = total_args_passed*wordSize;
1010 
1011   // Cut-out for having no stack args. Since up to 2 int/oop args are passed
1012   // in registers, we will occasionally have no stack args.
1013   int comp_words_on_stack = 0;
1014   if (comp_args_on_stack) {
1015     // Sig words on the stack are greater-than VMRegImpl::stack0. Those in
1016     // registers are below. By subtracting stack0, we either get a negative
1017     // number (all values in registers) or the maximum stack slot accessed.
1018 
1019     // Convert 4-byte c2 stack slots to words.
1020     comp_words_on_stack = round_to(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
1021     // Round up to miminum stack alignment, in wordSize.
1022     comp_words_on_stack = round_to(comp_words_on_stack, 2);
1023     __ resize_frame(-comp_words_on_stack * wordSize, R11_scratch1);
1024   }
1025 
1026   // Now generate the shuffle code.  Pick up all register args and move the
1027   // rest through register value=Z_R12.
1028   BLOCK_COMMENT("Shuffle arguments");
1029   for (int i = 0; i < total_args_passed; i++) {
1030     if (sig_bt[i] == T_VOID) {
1031       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
1032       continue;
1033     }
1034 
1035     // Pick up 0, 1 or 2 words from ld_ptr.
1036     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
1037             "scrambled load targets?");
1038     VMReg r_1 = regs[i].first();
1039     VMReg r_2 = regs[i].second();
1040     if (!r_1->is_valid()) {
1041       assert(!r_2->is_valid(), "");
1042       continue;
1043     }
1044     if (r_1->is_FloatRegister()) {
1045       if (!r_2->is_valid()) {
1046         __ lfs(r_1->as_FloatRegister(), ld_offset, ld_ptr);
1047         ld_offset-=wordSize;
1048       } else {
1049         // Skip the unused interpreter slot.
1050         __ lfd(r_1->as_FloatRegister(), ld_offset-wordSize, ld_ptr);
1051         ld_offset-=2*wordSize;
1052       }
1053     } else {
1054       Register r;
1055       if (r_1->is_stack()) {
1056         // Must do a memory to memory move thru "value".
1057         r = value_regs[value_regs_index];
1058         value_regs_index = (value_regs_index + 1) % num_value_regs;
1059       } else {
1060         r = r_1->as_Register();
1061       }
1062       if (!r_2->is_valid()) {
1063         // Not sure we need to do this but it shouldn't hurt.
1064         if (sig_bt[i] == T_OBJECT || sig_bt[i] == T_ADDRESS || sig_bt[i] == T_ARRAY) {
1065           __ ld(r, ld_offset, ld_ptr);
1066           ld_offset-=wordSize;
1067         } else {
1068           __ lwz(r, ld_offset, ld_ptr);
1069           ld_offset-=wordSize;
1070         }
1071       } else {
1072         // In 64bit, longs are given 2 64-bit slots in the interpreter, but the
1073         // data is passed in only 1 slot.
1074         if (sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
1075           ld_offset-=wordSize;
1076         }
1077         __ ld(r, ld_offset, ld_ptr);
1078         ld_offset-=wordSize;
1079       }
1080 
1081       if (r_1->is_stack()) {
1082         // Now store value where the compiler expects it
1083         int st_off = (r_1->reg2stack() + SharedRuntime::out_preserve_stack_slots())*VMRegImpl::stack_slot_size;
1084 
1085         if (sig_bt[i] == T_INT   || sig_bt[i] == T_FLOAT ||sig_bt[i] == T_BOOLEAN ||
1086             sig_bt[i] == T_SHORT || sig_bt[i] == T_CHAR  || sig_bt[i] == T_BYTE) {
1087           __ stw(r, st_off, R1_SP);
1088         } else {
1089           __ std(r, st_off, R1_SP);
1090         }
1091       }
1092     }
1093   }
1094 
1095   BLOCK_COMMENT("Store method");
1096   // Store method into thread->callee_target.
1097   // We might end up in handle_wrong_method if the callee is
1098   // deoptimized as we race thru here. If that happens we don't want
1099   // to take a safepoint because the caller frame will look
1100   // interpreted and arguments are now "compiled" so it is much better
1101   // to make this transition invisible to the stack walking
1102   // code. Unfortunately if we try and find the callee by normal means
1103   // a safepoint is possible. So we stash the desired callee in the
1104   // thread and the vm will find there should this case occur.
1105   __ std(R19_method, thread_(callee_target));
1106 
1107   // Jump to the compiled code just as if compiled code was doing it.
1108   __ bctr();
1109 }
1110 
1111 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
1112                                                             int total_args_passed,
1113                                                             int comp_args_on_stack,
1114                                                             const BasicType *sig_bt,
1115                                                             const VMRegPair *regs,
1116                                                             AdapterFingerPrint* fingerprint) {
1117   address i2c_entry;
1118   address c2i_unverified_entry;
1119   address c2i_entry;
1120 
1121 
1122   // entry: i2c
1123 
1124   __ align(CodeEntryAlignment);
1125   i2c_entry = __ pc();
1126   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
1127 
1128 
1129   // entry: c2i unverified
1130 
1131   __ align(CodeEntryAlignment);
1132   BLOCK_COMMENT("c2i unverified entry");
1133   c2i_unverified_entry = __ pc();
1134 
1135   // inline_cache contains a compiledICHolder
1136   const Register ic             = R19_method;
1137   const Register ic_klass       = R11_scratch1;
1138   const Register receiver_klass = R12_scratch2;
1139   const Register code           = R21_tmp1;
1140   const Register ientry         = R23_tmp3;
1141 
1142   assert_different_registers(ic, ic_klass, receiver_klass, R3_ARG1, code, ientry);
1143   assert(R11_scratch1 == R11, "need prologue scratch register");
1144 
1145   Label call_interpreter;
1146 
1147   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()),
1148          "klass offset should reach into any page");
1149   // Check for NULL argument if we don't have implicit null checks.
1150   if (!ImplicitNullChecks || !os::zero_page_read_protected()) {
1151     if (TrapBasedNullChecks) {
1152       __ trap_null_check(R3_ARG1);
1153     } else {
1154       Label valid;
1155       __ cmpdi(CCR0, R3_ARG1, 0);
1156       __ bne_predict_taken(CCR0, valid);
1157       // We have a null argument, branch to ic_miss_stub.
1158       __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
1159                        relocInfo::runtime_call_type);
1160       __ BIND(valid);
1161     }
1162   }
1163   // Assume argument is not NULL, load klass from receiver.
1164   __ load_klass(receiver_klass, R3_ARG1);
1165 
1166   __ ld(ic_klass, CompiledICHolder::holder_klass_offset(), ic);
1167 
1168   if (TrapBasedICMissChecks) {
1169     __ trap_ic_miss_check(receiver_klass, ic_klass);
1170   } else {
1171     Label valid;
1172     __ cmpd(CCR0, receiver_klass, ic_klass);
1173     __ beq_predict_taken(CCR0, valid);
1174     // We have an unexpected klass, branch to ic_miss_stub.
1175     __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
1176                      relocInfo::runtime_call_type);
1177     __ BIND(valid);
1178   }
1179 
1180   // Argument is valid and klass is as expected, continue.
1181 
1182   // Extract method from inline cache, verified entry point needs it.
1183   __ ld(R19_method, CompiledICHolder::holder_method_offset(), ic);
1184   assert(R19_method == ic, "the inline cache register is dead here");
1185 
1186   __ ld(code, method_(code));
1187   __ cmpdi(CCR0, code, 0);
1188   __ ld(ientry, method_(interpreter_entry)); // preloaded
1189   __ beq_predict_taken(CCR0, call_interpreter);
1190 
1191   // Branch to ic_miss_stub.
1192   __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type);
1193 
1194   // entry: c2i
1195 
1196   c2i_entry = gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, call_interpreter, ientry);
1197 
1198   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry);
1199 }
1200 
1201 #ifdef COMPILER2
1202 // An oop arg. Must pass a handle not the oop itself.
1203 static void object_move(MacroAssembler* masm,
1204                         int frame_size_in_slots,
1205                         OopMap* oop_map, int oop_handle_offset,
1206                         bool is_receiver, int* receiver_offset,
1207                         VMRegPair src, VMRegPair dst,
1208                         Register r_caller_sp, Register r_temp_1, Register r_temp_2) {
1209   assert(!is_receiver || (is_receiver && (*receiver_offset == -1)),
1210          "receiver has already been moved");
1211 
1212   // We must pass a handle. First figure out the location we use as a handle.
1213 
1214   if (src.first()->is_stack()) {
1215     // stack to stack or reg
1216 
1217     const Register r_handle = dst.first()->is_stack() ? r_temp_1 : dst.first()->as_Register();
1218     Label skip;
1219     const int oop_slot_in_callers_frame = reg2slot(src.first());
1220 
1221     guarantee(!is_receiver, "expecting receiver in register");
1222     oop_map->set_oop(VMRegImpl::stack2reg(oop_slot_in_callers_frame + frame_size_in_slots));
1223 
1224     __ addi(r_handle, r_caller_sp, reg2offset(src.first()));
1225     __ ld(  r_temp_2, reg2offset(src.first()), r_caller_sp);
1226     __ cmpdi(CCR0, r_temp_2, 0);
1227     __ bne(CCR0, skip);
1228     // Use a NULL handle if oop is NULL.
1229     __ li(r_handle, 0);
1230     __ bind(skip);
1231 
1232     if (dst.first()->is_stack()) {
1233       // stack to stack
1234       __ std(r_handle, reg2offset(dst.first()), R1_SP);
1235     } else {
1236       // stack to reg
1237       // Nothing to do, r_handle is already the dst register.
1238     }
1239   } else {
1240     // reg to stack or reg
1241     const Register r_oop      = src.first()->as_Register();
1242     const Register r_handle   = dst.first()->is_stack() ? r_temp_1 : dst.first()->as_Register();
1243     const int oop_slot        = (r_oop->encoding()-R3_ARG1->encoding()) * VMRegImpl::slots_per_word
1244                                 + oop_handle_offset; // in slots
1245     const int oop_offset = oop_slot * VMRegImpl::stack_slot_size;
1246     Label skip;
1247 
1248     if (is_receiver) {
1249       *receiver_offset = oop_offset;
1250     }
1251     oop_map->set_oop(VMRegImpl::stack2reg(oop_slot));
1252 
1253     __ std( r_oop,    oop_offset, R1_SP);
1254     __ addi(r_handle, R1_SP, oop_offset);
1255 
1256     __ cmpdi(CCR0, r_oop, 0);
1257     __ bne(CCR0, skip);
1258     // Use a NULL handle if oop is NULL.
1259     __ li(r_handle, 0);
1260     __ bind(skip);
1261 
1262     if (dst.first()->is_stack()) {
1263       // reg to stack
1264       __ std(r_handle, reg2offset(dst.first()), R1_SP);
1265     } else {
1266       // reg to reg
1267       // Nothing to do, r_handle is already the dst register.
1268     }
1269   }
1270 }
1271 
1272 static void int_move(MacroAssembler*masm,
1273                      VMRegPair src, VMRegPair dst,
1274                      Register r_caller_sp, Register r_temp) {
1275   assert(src.first()->is_valid(), "incoming must be int");
1276   assert(dst.first()->is_valid() && dst.second() == dst.first()->next(), "outgoing must be long");
1277 
1278   if (src.first()->is_stack()) {
1279     if (dst.first()->is_stack()) {
1280       // stack to stack
1281       __ lwa(r_temp, reg2offset(src.first()), r_caller_sp);
1282       __ std(r_temp, reg2offset(dst.first()), R1_SP);
1283     } else {
1284       // stack to reg
1285       __ lwa(dst.first()->as_Register(), reg2offset(src.first()), r_caller_sp);
1286     }
1287   } else if (dst.first()->is_stack()) {
1288     // reg to stack
1289     __ extsw(r_temp, src.first()->as_Register());
1290     __ std(r_temp, reg2offset(dst.first()), R1_SP);
1291   } else {
1292     // reg to reg
1293     __ extsw(dst.first()->as_Register(), src.first()->as_Register());
1294   }
1295 }
1296 
1297 static void long_move(MacroAssembler*masm,
1298                       VMRegPair src, VMRegPair dst,
1299                       Register r_caller_sp, Register r_temp) {
1300   assert(src.first()->is_valid() && src.second() == src.first()->next(), "incoming must be long");
1301   assert(dst.first()->is_valid() && dst.second() == dst.first()->next(), "outgoing must be long");
1302 
1303   if (src.first()->is_stack()) {
1304     if (dst.first()->is_stack()) {
1305       // stack to stack
1306       __ ld( r_temp, reg2offset(src.first()), r_caller_sp);
1307       __ std(r_temp, reg2offset(dst.first()), R1_SP);
1308     } else {
1309       // stack to reg
1310       __ ld(dst.first()->as_Register(), reg2offset(src.first()), r_caller_sp);
1311     }
1312   } else if (dst.first()->is_stack()) {
1313     // reg to stack
1314     __ std(src.first()->as_Register(), reg2offset(dst.first()), R1_SP);
1315   } else {
1316     // reg to reg
1317     if (dst.first()->as_Register() != src.first()->as_Register())
1318       __ mr(dst.first()->as_Register(), src.first()->as_Register());
1319   }
1320 }
1321 
1322 static void float_move(MacroAssembler*masm,
1323                        VMRegPair src, VMRegPair dst,
1324                        Register r_caller_sp, Register r_temp) {
1325   assert(src.first()->is_valid() && !src.second()->is_valid(), "incoming must be float");
1326   assert(dst.first()->is_valid() && !dst.second()->is_valid(), "outgoing must be float");
1327 
1328   if (src.first()->is_stack()) {
1329     if (dst.first()->is_stack()) {
1330       // stack to stack
1331       __ lwz(r_temp, reg2offset(src.first()), r_caller_sp);
1332       __ stw(r_temp, reg2offset(dst.first()), R1_SP);
1333     } else {
1334       // stack to reg
1335       __ lfs(dst.first()->as_FloatRegister(), reg2offset(src.first()), r_caller_sp);
1336     }
1337   } else if (dst.first()->is_stack()) {
1338     // reg to stack
1339     __ stfs(src.first()->as_FloatRegister(), reg2offset(dst.first()), R1_SP);
1340   } else {
1341     // reg to reg
1342     if (dst.first()->as_FloatRegister() != src.first()->as_FloatRegister())
1343       __ fmr(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
1344   }
1345 }
1346 
1347 static void double_move(MacroAssembler*masm,
1348                         VMRegPair src, VMRegPair dst,
1349                         Register r_caller_sp, Register r_temp) {
1350   assert(src.first()->is_valid() && src.second() == src.first()->next(), "incoming must be double");
1351   assert(dst.first()->is_valid() && dst.second() == dst.first()->next(), "outgoing must be double");
1352 
1353   if (src.first()->is_stack()) {
1354     if (dst.first()->is_stack()) {
1355       // stack to stack
1356       __ ld( r_temp, reg2offset(src.first()), r_caller_sp);
1357       __ std(r_temp, reg2offset(dst.first()), R1_SP);
1358     } else {
1359       // stack to reg
1360       __ lfd(dst.first()->as_FloatRegister(), reg2offset(src.first()), r_caller_sp);
1361     }
1362   } else if (dst.first()->is_stack()) {
1363     // reg to stack
1364     __ stfd(src.first()->as_FloatRegister(), reg2offset(dst.first()), R1_SP);
1365   } else {
1366     // reg to reg
1367     if (dst.first()->as_FloatRegister() != src.first()->as_FloatRegister())
1368       __ fmr(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
1369   }
1370 }
1371 
1372 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1373   switch (ret_type) {
1374     case T_BOOLEAN:
1375     case T_CHAR:
1376     case T_BYTE:
1377     case T_SHORT:
1378     case T_INT:
1379       __ stw (R3_RET,  frame_slots*VMRegImpl::stack_slot_size, R1_SP);
1380       break;
1381     case T_ARRAY:
1382     case T_OBJECT:
1383     case T_LONG:
1384       __ std (R3_RET,  frame_slots*VMRegImpl::stack_slot_size, R1_SP);
1385       break;
1386     case T_FLOAT:
1387       __ stfs(F1_RET, frame_slots*VMRegImpl::stack_slot_size, R1_SP);
1388       break;
1389     case T_DOUBLE:
1390       __ stfd(F1_RET, frame_slots*VMRegImpl::stack_slot_size, R1_SP);
1391       break;
1392     case T_VOID:
1393       break;
1394     default:
1395       ShouldNotReachHere();
1396       break;
1397   }
1398 }
1399 
1400 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1401   switch (ret_type) {
1402     case T_BOOLEAN:
1403     case T_CHAR:
1404     case T_BYTE:
1405     case T_SHORT:
1406     case T_INT:
1407       __ lwz(R3_RET,  frame_slots*VMRegImpl::stack_slot_size, R1_SP);
1408       break;
1409     case T_ARRAY:
1410     case T_OBJECT:
1411     case T_LONG:
1412       __ ld (R3_RET,  frame_slots*VMRegImpl::stack_slot_size, R1_SP);
1413       break;
1414     case T_FLOAT:
1415       __ lfs(F1_RET, frame_slots*VMRegImpl::stack_slot_size, R1_SP);
1416       break;
1417     case T_DOUBLE:
1418       __ lfd(F1_RET, frame_slots*VMRegImpl::stack_slot_size, R1_SP);
1419       break;
1420     case T_VOID:
1421       break;
1422     default:
1423       ShouldNotReachHere();
1424       break;
1425   }
1426 }
1427 
1428 static void save_or_restore_arguments(MacroAssembler* masm,
1429                                       const int stack_slots,
1430                                       const int total_in_args,
1431                                       const int arg_save_area,
1432                                       OopMap* map,
1433                                       VMRegPair* in_regs,
1434                                       BasicType* in_sig_bt) {
1435   // If map is non-NULL then the code should store the values,
1436   // otherwise it should load them.
1437   int slot = arg_save_area;
1438   // Save down double word first.
1439   for (int i = 0; i < total_in_args; i++) {
1440     if (in_regs[i].first()->is_FloatRegister() && in_sig_bt[i] == T_DOUBLE) {
1441       int offset = slot * VMRegImpl::stack_slot_size;
1442       slot += VMRegImpl::slots_per_word;
1443       assert(slot <= stack_slots, "overflow (after DOUBLE stack slot)");
1444       if (map != NULL) {
1445         __ stfd(in_regs[i].first()->as_FloatRegister(), offset, R1_SP);
1446       } else {
1447         __ lfd(in_regs[i].first()->as_FloatRegister(), offset, R1_SP);
1448       }
1449     } else if (in_regs[i].first()->is_Register() &&
1450         (in_sig_bt[i] == T_LONG || in_sig_bt[i] == T_ARRAY)) {
1451       int offset = slot * VMRegImpl::stack_slot_size;
1452       if (map != NULL) {
1453         __ std(in_regs[i].first()->as_Register(), offset, R1_SP);
1454         if (in_sig_bt[i] == T_ARRAY) {
1455           map->set_oop(VMRegImpl::stack2reg(slot));
1456         }
1457       } else {
1458         __ ld(in_regs[i].first()->as_Register(), offset, R1_SP);
1459       }
1460       slot += VMRegImpl::slots_per_word;
1461       assert(slot <= stack_slots, "overflow (after LONG/ARRAY stack slot)");
1462     }
1463   }
1464   // Save or restore single word registers.
1465   for (int i = 0; i < total_in_args; i++) {
1466     // PPC64: pass ints as longs: must only deal with floats here.
1467     if (in_regs[i].first()->is_FloatRegister()) {
1468       if (in_sig_bt[i] == T_FLOAT) {
1469         int offset = slot * VMRegImpl::stack_slot_size;
1470         slot++;
1471         assert(slot <= stack_slots, "overflow (after FLOAT stack slot)");
1472         if (map != NULL) {
1473           __ stfs(in_regs[i].first()->as_FloatRegister(), offset, R1_SP);
1474         } else {
1475           __ lfs(in_regs[i].first()->as_FloatRegister(), offset, R1_SP);
1476         }
1477       }
1478     } else if (in_regs[i].first()->is_stack()) {
1479       if (in_sig_bt[i] == T_ARRAY && map != NULL) {
1480         int offset_in_older_frame = in_regs[i].first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1481         map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
1482       }
1483     }
1484   }
1485 }
1486 
1487 // Check GC_locker::needs_gc and enter the runtime if it's true. This
1488 // keeps a new JNI critical region from starting until a GC has been
1489 // forced. Save down any oops in registers and describe them in an
1490 // OopMap.
1491 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1492                                                const int stack_slots,
1493                                                const int total_in_args,
1494                                                const int arg_save_area,
1495                                                OopMapSet* oop_maps,
1496                                                VMRegPair* in_regs,
1497                                                BasicType* in_sig_bt,
1498                                                Register tmp_reg ) {
1499   __ block_comment("check GC_locker::needs_gc");
1500   Label cont;
1501   __ lbz(tmp_reg, (RegisterOrConstant)(intptr_t)GC_locker::needs_gc_address());
1502   __ cmplwi(CCR0, tmp_reg, 0);
1503   __ beq(CCR0, cont);
1504 
1505   // Save down any values that are live in registers and call into the
1506   // runtime to halt for a GC.
1507   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1508   save_or_restore_arguments(masm, stack_slots, total_in_args,
1509                             arg_save_area, map, in_regs, in_sig_bt);
1510 
1511   __ mr(R3_ARG1, R16_thread);
1512   __ set_last_Java_frame(R1_SP, noreg);
1513 
1514   __ block_comment("block_for_jni_critical");
1515   address entry_point = CAST_FROM_FN_PTR(address, SharedRuntime::block_for_jni_critical);
1516 #if defined(ABI_ELFv2)
1517   __ call_c(entry_point, relocInfo::runtime_call_type);
1518 #else
1519   __ call_c(CAST_FROM_FN_PTR(FunctionDescriptor*, entry_point), relocInfo::runtime_call_type);
1520 #endif
1521   address start           = __ pc() - __ offset(),
1522           calls_return_pc = __ last_calls_return_pc();
1523   oop_maps->add_gc_map(calls_return_pc - start, map);
1524 
1525   __ reset_last_Java_frame();
1526 
1527   // Reload all the register arguments.
1528   save_or_restore_arguments(masm, stack_slots, total_in_args,
1529                             arg_save_area, NULL, in_regs, in_sig_bt);
1530 
1531   __ BIND(cont);
1532 
1533 #ifdef ASSERT
1534   if (StressCriticalJNINatives) {
1535     // Stress register saving.
1536     OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1537     save_or_restore_arguments(masm, stack_slots, total_in_args,
1538                               arg_save_area, map, in_regs, in_sig_bt);
1539     // Destroy argument registers.
1540     for (int i = 0; i < total_in_args; i++) {
1541       if (in_regs[i].first()->is_Register()) {
1542         const Register reg = in_regs[i].first()->as_Register();
1543         __ neg(reg, reg);
1544       } else if (in_regs[i].first()->is_FloatRegister()) {
1545         __ fneg(in_regs[i].first()->as_FloatRegister(), in_regs[i].first()->as_FloatRegister());
1546       }
1547     }
1548 
1549     save_or_restore_arguments(masm, stack_slots, total_in_args,
1550                               arg_save_area, NULL, in_regs, in_sig_bt);
1551   }
1552 #endif
1553 }
1554 
1555 static void move_ptr(MacroAssembler* masm, VMRegPair src, VMRegPair dst, Register r_caller_sp, Register r_temp) {
1556   if (src.first()->is_stack()) {
1557     if (dst.first()->is_stack()) {
1558       // stack to stack
1559       __ ld(r_temp, reg2offset(src.first()), r_caller_sp);
1560       __ std(r_temp, reg2offset(dst.first()), R1_SP);
1561     } else {
1562       // stack to reg
1563       __ ld(dst.first()->as_Register(), reg2offset(src.first()), r_caller_sp);
1564     }
1565   } else if (dst.first()->is_stack()) {
1566     // reg to stack
1567     __ std(src.first()->as_Register(), reg2offset(dst.first()), R1_SP);
1568   } else {
1569     if (dst.first() != src.first()) {
1570       __ mr(dst.first()->as_Register(), src.first()->as_Register());
1571     }
1572   }
1573 }
1574 
1575 // Unpack an array argument into a pointer to the body and the length
1576 // if the array is non-null, otherwise pass 0 for both.
1577 static void unpack_array_argument(MacroAssembler* masm, VMRegPair reg, BasicType in_elem_type,
1578                                   VMRegPair body_arg, VMRegPair length_arg, Register r_caller_sp,
1579                                   Register tmp_reg, Register tmp2_reg) {
1580   assert(!body_arg.first()->is_Register() || body_arg.first()->as_Register() != tmp_reg,
1581          "possible collision");
1582   assert(!length_arg.first()->is_Register() || length_arg.first()->as_Register() != tmp_reg,
1583          "possible collision");
1584 
1585   // Pass the length, ptr pair.
1586   Label set_out_args;
1587   VMRegPair tmp, tmp2;
1588   tmp.set_ptr(tmp_reg->as_VMReg());
1589   tmp2.set_ptr(tmp2_reg->as_VMReg());
1590   if (reg.first()->is_stack()) {
1591     // Load the arg up from the stack.
1592     move_ptr(masm, reg, tmp, r_caller_sp, /*unused*/ R0);
1593     reg = tmp;
1594   }
1595   __ li(tmp2_reg, 0); // Pass zeros if Array=null.
1596   if (tmp_reg != reg.first()->as_Register()) __ li(tmp_reg, 0);
1597   __ cmpdi(CCR0, reg.first()->as_Register(), 0);
1598   __ beq(CCR0, set_out_args);
1599   __ lwa(tmp2_reg, arrayOopDesc::length_offset_in_bytes(), reg.first()->as_Register());
1600   __ addi(tmp_reg, reg.first()->as_Register(), arrayOopDesc::base_offset_in_bytes(in_elem_type));
1601   __ bind(set_out_args);
1602   move_ptr(masm, tmp, body_arg, r_caller_sp, /*unused*/ R0);
1603   move_ptr(masm, tmp2, length_arg, r_caller_sp, /*unused*/ R0); // Same as move32_64 on PPC64.
1604 }
1605 
1606 static void verify_oop_args(MacroAssembler* masm,
1607                             methodHandle method,
1608                             const BasicType* sig_bt,
1609                             const VMRegPair* regs) {
1610   Register temp_reg = R19_method;  // not part of any compiled calling seq
1611   if (VerifyOops) {
1612     for (int i = 0; i < method->size_of_parameters(); i++) {
1613       if (sig_bt[i] == T_OBJECT ||
1614           sig_bt[i] == T_ARRAY) {
1615         VMReg r = regs[i].first();
1616         assert(r->is_valid(), "bad oop arg");
1617         if (r->is_stack()) {
1618           __ ld(temp_reg, reg2offset(r), R1_SP);
1619           __ verify_oop(temp_reg);
1620         } else {
1621           __ verify_oop(r->as_Register());
1622         }
1623       }
1624     }
1625   }
1626 }
1627 
1628 static void gen_special_dispatch(MacroAssembler* masm,
1629                                  methodHandle method,
1630                                  const BasicType* sig_bt,
1631                                  const VMRegPair* regs) {
1632   verify_oop_args(masm, method, sig_bt, regs);
1633   vmIntrinsics::ID iid = method->intrinsic_id();
1634 
1635   // Now write the args into the outgoing interpreter space
1636   bool     has_receiver   = false;
1637   Register receiver_reg   = noreg;
1638   int      member_arg_pos = -1;
1639   Register member_reg     = noreg;
1640   int      ref_kind       = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1641   if (ref_kind != 0) {
1642     member_arg_pos = method->size_of_parameters() - 1;  // trailing MemberName argument
1643     member_reg = R19_method;  // known to be free at this point
1644     has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1645   } else if (iid == vmIntrinsics::_invokeBasic) {
1646     has_receiver = true;
1647   } else {
1648     fatal("unexpected intrinsic id %d", iid);
1649   }
1650 
1651   if (member_reg != noreg) {
1652     // Load the member_arg into register, if necessary.
1653     SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1654     VMReg r = regs[member_arg_pos].first();
1655     if (r->is_stack()) {
1656       __ ld(member_reg, reg2offset(r), R1_SP);
1657     } else {
1658       // no data motion is needed
1659       member_reg = r->as_Register();
1660     }
1661   }
1662 
1663   if (has_receiver) {
1664     // Make sure the receiver is loaded into a register.
1665     assert(method->size_of_parameters() > 0, "oob");
1666     assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1667     VMReg r = regs[0].first();
1668     assert(r->is_valid(), "bad receiver arg");
1669     if (r->is_stack()) {
1670       // Porting note:  This assumes that compiled calling conventions always
1671       // pass the receiver oop in a register.  If this is not true on some
1672       // platform, pick a temp and load the receiver from stack.
1673       fatal("receiver always in a register");
1674       receiver_reg = R11_scratch1;  // TODO (hs24): is R11_scratch1 really free at this point?
1675       __ ld(receiver_reg, reg2offset(r), R1_SP);
1676     } else {
1677       // no data motion is needed
1678       receiver_reg = r->as_Register();
1679     }
1680   }
1681 
1682   // Figure out which address we are really jumping to:
1683   MethodHandles::generate_method_handle_dispatch(masm, iid,
1684                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
1685 }
1686 
1687 #endif // COMPILER2
1688 
1689 // ---------------------------------------------------------------------------
1690 // Generate a native wrapper for a given method. The method takes arguments
1691 // in the Java compiled code convention, marshals them to the native
1692 // convention (handlizes oops, etc), transitions to native, makes the call,
1693 // returns to java state (possibly blocking), unhandlizes any result and
1694 // returns.
1695 //
1696 // Critical native functions are a shorthand for the use of
1697 // GetPrimtiveArrayCritical and disallow the use of any other JNI
1698 // functions.  The wrapper is expected to unpack the arguments before
1699 // passing them to the callee and perform checks before and after the
1700 // native call to ensure that they GC_locker
1701 // lock_critical/unlock_critical semantics are followed.  Some other
1702 // parts of JNI setup are skipped like the tear down of the JNI handle
1703 // block and the check for pending exceptions it's impossible for them
1704 // to be thrown.
1705 //
1706 // They are roughly structured like this:
1707 //   if (GC_locker::needs_gc())
1708 //     SharedRuntime::block_for_jni_critical();
1709 //   tranistion to thread_in_native
1710 //   unpack arrray arguments and call native entry point
1711 //   check for safepoint in progress
1712 //   check if any thread suspend flags are set
1713 //     call into JVM and possible unlock the JNI critical
1714 //     if a GC was suppressed while in the critical native.
1715 //   transition back to thread_in_Java
1716 //   return to caller
1717 //
1718 nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
1719                                                 methodHandle method,
1720                                                 int compile_id,
1721                                                 BasicType *in_sig_bt,
1722                                                 VMRegPair *in_regs,
1723                                                 BasicType ret_type) {
1724 #ifdef COMPILER2
1725   if (method->is_method_handle_intrinsic()) {
1726     vmIntrinsics::ID iid = method->intrinsic_id();
1727     intptr_t start = (intptr_t)__ pc();
1728     int vep_offset = ((intptr_t)__ pc()) - start;
1729     gen_special_dispatch(masm,
1730                          method,
1731                          in_sig_bt,
1732                          in_regs);
1733     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
1734     __ flush();
1735     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
1736     return nmethod::new_native_nmethod(method,
1737                                        compile_id,
1738                                        masm->code(),
1739                                        vep_offset,
1740                                        frame_complete,
1741                                        stack_slots / VMRegImpl::slots_per_word,
1742                                        in_ByteSize(-1),
1743                                        in_ByteSize(-1),
1744                                        (OopMapSet*)NULL);
1745   }
1746 
1747   bool is_critical_native = true;
1748   address native_func = method->critical_native_function();
1749   if (native_func == NULL) {
1750     native_func = method->native_function();
1751     is_critical_native = false;
1752   }
1753   assert(native_func != NULL, "must have function");
1754 
1755   // First, create signature for outgoing C call
1756   // --------------------------------------------------------------------------
1757 
1758   int total_in_args = method->size_of_parameters();
1759   // We have received a description of where all the java args are located
1760   // on entry to the wrapper. We need to convert these args to where
1761   // the jni function will expect them. To figure out where they go
1762   // we convert the java signature to a C signature by inserting
1763   // the hidden arguments as arg[0] and possibly arg[1] (static method)
1764 
1765   // Calculate the total number of C arguments and create arrays for the
1766   // signature and the outgoing registers.
1767   // On ppc64, we have two arrays for the outgoing registers, because
1768   // some floating-point arguments must be passed in registers _and_
1769   // in stack locations.
1770   bool method_is_static = method->is_static();
1771   int  total_c_args     = total_in_args;
1772 
1773   if (!is_critical_native) {
1774     int n_hidden_args = method_is_static ? 2 : 1;
1775     total_c_args += n_hidden_args;
1776   } else {
1777     // No JNIEnv*, no this*, but unpacked arrays (base+length).
1778     for (int i = 0; i < total_in_args; i++) {
1779       if (in_sig_bt[i] == T_ARRAY) {
1780         total_c_args++;
1781       }
1782     }
1783   }
1784 
1785   BasicType *out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1786   VMRegPair *out_regs   = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1787   VMRegPair *out_regs2  = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1788   BasicType* in_elem_bt = NULL;
1789 
1790   // Create the signature for the C call:
1791   //   1) add the JNIEnv*
1792   //   2) add the class if the method is static
1793   //   3) copy the rest of the incoming signature (shifted by the number of
1794   //      hidden arguments).
1795 
1796   int argc = 0;
1797   if (!is_critical_native) {
1798     out_sig_bt[argc++] = T_ADDRESS;
1799     if (method->is_static()) {
1800       out_sig_bt[argc++] = T_OBJECT;
1801     }
1802 
1803     for (int i = 0; i < total_in_args ; i++ ) {
1804       out_sig_bt[argc++] = in_sig_bt[i];
1805     }
1806   } else {
1807     Thread* THREAD = Thread::current();
1808     in_elem_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1809     SignatureStream ss(method->signature());
1810     int o = 0;
1811     for (int i = 0; i < total_in_args ; i++, o++) {
1812       if (in_sig_bt[i] == T_ARRAY) {
1813         // Arrays are passed as int, elem* pair
1814         Symbol* atype = ss.as_symbol(CHECK_NULL);
1815         const char* at = atype->as_C_string();
1816         if (strlen(at) == 2) {
1817           assert(at[0] == '[', "must be");
1818           switch (at[1]) {
1819             case 'B': in_elem_bt[o] = T_BYTE; break;
1820             case 'C': in_elem_bt[o] = T_CHAR; break;
1821             case 'D': in_elem_bt[o] = T_DOUBLE; break;
1822             case 'F': in_elem_bt[o] = T_FLOAT; break;
1823             case 'I': in_elem_bt[o] = T_INT; break;
1824             case 'J': in_elem_bt[o] = T_LONG; break;
1825             case 'S': in_elem_bt[o] = T_SHORT; break;
1826             case 'Z': in_elem_bt[o] = T_BOOLEAN; break;
1827             default: ShouldNotReachHere();
1828           }
1829         }
1830       } else {
1831         in_elem_bt[o] = T_VOID;
1832       }
1833       if (in_sig_bt[i] != T_VOID) {
1834         assert(in_sig_bt[i] == ss.type(), "must match");
1835         ss.next();
1836       }
1837     }
1838 
1839     for (int i = 0; i < total_in_args ; i++ ) {
1840       if (in_sig_bt[i] == T_ARRAY) {
1841         // Arrays are passed as int, elem* pair.
1842         out_sig_bt[argc++] = T_INT;
1843         out_sig_bt[argc++] = T_ADDRESS;
1844       } else {
1845         out_sig_bt[argc++] = in_sig_bt[i];
1846       }
1847     }
1848   }
1849 
1850 
1851   // Compute the wrapper's frame size.
1852   // --------------------------------------------------------------------------
1853 
1854   // Now figure out where the args must be stored and how much stack space
1855   // they require.
1856   //
1857   // Compute framesize for the wrapper. We need to handlize all oops in
1858   // incoming registers.
1859   //
1860   // Calculate the total number of stack slots we will need:
1861   //   1) abi requirements
1862   //   2) outgoing arguments
1863   //   3) space for inbound oop handle area
1864   //   4) space for handlizing a klass if static method
1865   //   5) space for a lock if synchronized method
1866   //   6) workspace for saving return values, int <-> float reg moves, etc.
1867   //   7) alignment
1868   //
1869   // Layout of the native wrapper frame:
1870   // (stack grows upwards, memory grows downwards)
1871   //
1872   // NW     [ABI_REG_ARGS]             <-- 1) R1_SP
1873   //        [outgoing arguments]       <-- 2) R1_SP + out_arg_slot_offset
1874   //        [oopHandle area]           <-- 3) R1_SP + oop_handle_offset (save area for critical natives)
1875   //        klass                      <-- 4) R1_SP + klass_offset
1876   //        lock                       <-- 5) R1_SP + lock_offset
1877   //        [workspace]                <-- 6) R1_SP + workspace_offset
1878   //        [alignment] (optional)     <-- 7)
1879   // caller [JIT_TOP_ABI_48]           <-- r_callers_sp
1880   //
1881   // - *_slot_offset Indicates offset from SP in number of stack slots.
1882   // - *_offset      Indicates offset from SP in bytes.
1883 
1884   int stack_slots = c_calling_convention(out_sig_bt, out_regs, out_regs2, total_c_args) // 1+2)
1885                   + SharedRuntime::out_preserve_stack_slots(); // See c_calling_convention.
1886 
1887   // Now the space for the inbound oop handle area.
1888   int total_save_slots = num_java_iarg_registers * VMRegImpl::slots_per_word;
1889   if (is_critical_native) {
1890     // Critical natives may have to call out so they need a save area
1891     // for register arguments.
1892     int double_slots = 0;
1893     int single_slots = 0;
1894     for (int i = 0; i < total_in_args; i++) {
1895       if (in_regs[i].first()->is_Register()) {
1896         const Register reg = in_regs[i].first()->as_Register();
1897         switch (in_sig_bt[i]) {
1898           case T_BOOLEAN:
1899           case T_BYTE:
1900           case T_SHORT:
1901           case T_CHAR:
1902           case T_INT:
1903           // Fall through.
1904           case T_ARRAY:
1905           case T_LONG: double_slots++; break;
1906           default:  ShouldNotReachHere();
1907         }
1908       } else if (in_regs[i].first()->is_FloatRegister()) {
1909         switch (in_sig_bt[i]) {
1910           case T_FLOAT:  single_slots++; break;
1911           case T_DOUBLE: double_slots++; break;
1912           default:  ShouldNotReachHere();
1913         }
1914       }
1915     }
1916     total_save_slots = double_slots * 2 + round_to(single_slots, 2); // round to even
1917   }
1918 
1919   int oop_handle_slot_offset = stack_slots;
1920   stack_slots += total_save_slots;                                                // 3)
1921 
1922   int klass_slot_offset = 0;
1923   int klass_offset      = -1;
1924   if (method_is_static && !is_critical_native) {                                  // 4)
1925     klass_slot_offset  = stack_slots;
1926     klass_offset       = klass_slot_offset * VMRegImpl::stack_slot_size;
1927     stack_slots       += VMRegImpl::slots_per_word;
1928   }
1929 
1930   int lock_slot_offset = 0;
1931   int lock_offset      = -1;
1932   if (method->is_synchronized()) {                                                // 5)
1933     lock_slot_offset   = stack_slots;
1934     lock_offset        = lock_slot_offset * VMRegImpl::stack_slot_size;
1935     stack_slots       += VMRegImpl::slots_per_word;
1936   }
1937 
1938   int workspace_slot_offset = stack_slots;                                        // 6)
1939   stack_slots         += 2;
1940 
1941   // Now compute actual number of stack words we need.
1942   // Rounding to make stack properly aligned.
1943   stack_slots = round_to(stack_slots,                                             // 7)
1944                          frame::alignment_in_bytes / VMRegImpl::stack_slot_size);
1945   int frame_size_in_bytes = stack_slots * VMRegImpl::stack_slot_size;
1946 
1947 
1948   // Now we can start generating code.
1949   // --------------------------------------------------------------------------
1950 
1951   intptr_t start_pc = (intptr_t)__ pc();
1952   intptr_t vep_start_pc;
1953   intptr_t frame_done_pc;
1954   intptr_t oopmap_pc;
1955 
1956   Label    ic_miss;
1957   Label    handle_pending_exception;
1958 
1959   Register r_callers_sp = R21;
1960   Register r_temp_1     = R22;
1961   Register r_temp_2     = R23;
1962   Register r_temp_3     = R24;
1963   Register r_temp_4     = R25;
1964   Register r_temp_5     = R26;
1965   Register r_temp_6     = R27;
1966   Register r_return_pc  = R28;
1967 
1968   Register r_carg1_jnienv        = noreg;
1969   Register r_carg2_classorobject = noreg;
1970   if (!is_critical_native) {
1971     r_carg1_jnienv        = out_regs[0].first()->as_Register();
1972     r_carg2_classorobject = out_regs[1].first()->as_Register();
1973   }
1974 
1975 
1976   // Generate the Unverified Entry Point (UEP).
1977   // --------------------------------------------------------------------------
1978   assert(start_pc == (intptr_t)__ pc(), "uep must be at start");
1979 
1980   // Check ic: object class == cached class?
1981   if (!method_is_static) {
1982   Register ic = as_Register(Matcher::inline_cache_reg_encode());
1983   Register receiver_klass = r_temp_1;
1984 
1985   __ cmpdi(CCR0, R3_ARG1, 0);
1986   __ beq(CCR0, ic_miss);
1987   __ verify_oop(R3_ARG1);
1988   __ load_klass(receiver_klass, R3_ARG1);
1989 
1990   __ cmpd(CCR0, receiver_klass, ic);
1991   __ bne(CCR0, ic_miss);
1992   }
1993 
1994 
1995   // Generate the Verified Entry Point (VEP).
1996   // --------------------------------------------------------------------------
1997   vep_start_pc = (intptr_t)__ pc();
1998 
1999   __ save_LR_CR(r_temp_1);
2000   __ generate_stack_overflow_check(frame_size_in_bytes); // Check before creating frame.
2001   __ mr(r_callers_sp, R1_SP);                            // Remember frame pointer.
2002   __ push_frame(frame_size_in_bytes, r_temp_1);          // Push the c2n adapter's frame.
2003   frame_done_pc = (intptr_t)__ pc();
2004 
2005   __ verify_thread();
2006 
2007   // Native nmethod wrappers never take possesion of the oop arguments.
2008   // So the caller will gc the arguments.
2009   // The only thing we need an oopMap for is if the call is static.
2010   //
2011   // An OopMap for lock (and class if static), and one for the VM call itself.
2012   OopMapSet *oop_maps = new OopMapSet();
2013   OopMap    *oop_map  = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
2014 
2015   if (is_critical_native) {
2016     check_needs_gc_for_critical_native(masm, stack_slots, total_in_args, oop_handle_slot_offset, oop_maps, in_regs, in_sig_bt, r_temp_1);
2017   }
2018 
2019   // Move arguments from register/stack to register/stack.
2020   // --------------------------------------------------------------------------
2021   //
2022   // We immediately shuffle the arguments so that for any vm call we have
2023   // to make from here on out (sync slow path, jvmti, etc.) we will have
2024   // captured the oops from our caller and have a valid oopMap for them.
2025   //
2026   // Natives require 1 or 2 extra arguments over the normal ones: the JNIEnv*
2027   // (derived from JavaThread* which is in R16_thread) and, if static,
2028   // the class mirror instead of a receiver. This pretty much guarantees that
2029   // register layout will not match. We ignore these extra arguments during
2030   // the shuffle. The shuffle is described by the two calling convention
2031   // vectors we have in our possession. We simply walk the java vector to
2032   // get the source locations and the c vector to get the destinations.
2033 
2034   // Record sp-based slot for receiver on stack for non-static methods.
2035   int receiver_offset = -1;
2036 
2037   // We move the arguments backward because the floating point registers
2038   // destination will always be to a register with a greater or equal
2039   // register number or the stack.
2040   //   in  is the index of the incoming Java arguments
2041   //   out is the index of the outgoing C arguments
2042 
2043 #ifdef ASSERT
2044   bool reg_destroyed[RegisterImpl::number_of_registers];
2045   bool freg_destroyed[FloatRegisterImpl::number_of_registers];
2046   for (int r = 0 ; r < RegisterImpl::number_of_registers ; r++) {
2047     reg_destroyed[r] = false;
2048   }
2049   for (int f = 0 ; f < FloatRegisterImpl::number_of_registers ; f++) {
2050     freg_destroyed[f] = false;
2051   }
2052 #endif // ASSERT
2053 
2054   for (int in = total_in_args - 1, out = total_c_args - 1; in >= 0 ; in--, out--) {
2055 
2056 #ifdef ASSERT
2057     if (in_regs[in].first()->is_Register()) {
2058       assert(!reg_destroyed[in_regs[in].first()->as_Register()->encoding()], "ack!");
2059     } else if (in_regs[in].first()->is_FloatRegister()) {
2060       assert(!freg_destroyed[in_regs[in].first()->as_FloatRegister()->encoding()], "ack!");
2061     }
2062     if (out_regs[out].first()->is_Register()) {
2063       reg_destroyed[out_regs[out].first()->as_Register()->encoding()] = true;
2064     } else if (out_regs[out].first()->is_FloatRegister()) {
2065       freg_destroyed[out_regs[out].first()->as_FloatRegister()->encoding()] = true;
2066     }
2067     if (out_regs2[out].first()->is_Register()) {
2068       reg_destroyed[out_regs2[out].first()->as_Register()->encoding()] = true;
2069     } else if (out_regs2[out].first()->is_FloatRegister()) {
2070       freg_destroyed[out_regs2[out].first()->as_FloatRegister()->encoding()] = true;
2071     }
2072 #endif // ASSERT
2073 
2074     switch (in_sig_bt[in]) {
2075       case T_BOOLEAN:
2076       case T_CHAR:
2077       case T_BYTE:
2078       case T_SHORT:
2079       case T_INT:
2080         // Move int and do sign extension.
2081         int_move(masm, in_regs[in], out_regs[out], r_callers_sp, r_temp_1);
2082         break;
2083       case T_LONG:
2084         long_move(masm, in_regs[in], out_regs[out], r_callers_sp, r_temp_1);
2085         break;
2086       case T_ARRAY:
2087         if (is_critical_native) {
2088           int body_arg = out;
2089           out -= 1; // Point to length arg.
2090           unpack_array_argument(masm, in_regs[in], in_elem_bt[in], out_regs[body_arg], out_regs[out],
2091                                 r_callers_sp, r_temp_1, r_temp_2);
2092           break;
2093         }
2094       case T_OBJECT:
2095         assert(!is_critical_native, "no oop arguments");
2096         object_move(masm, stack_slots,
2097                     oop_map, oop_handle_slot_offset,
2098                     ((in == 0) && (!method_is_static)), &receiver_offset,
2099                     in_regs[in], out_regs[out],
2100                     r_callers_sp, r_temp_1, r_temp_2);
2101         break;
2102       case T_VOID:
2103         break;
2104       case T_FLOAT:
2105         float_move(masm, in_regs[in], out_regs[out], r_callers_sp, r_temp_1);
2106         if (out_regs2[out].first()->is_valid()) {
2107           float_move(masm, in_regs[in], out_regs2[out], r_callers_sp, r_temp_1);
2108         }
2109         break;
2110       case T_DOUBLE:
2111         double_move(masm, in_regs[in], out_regs[out], r_callers_sp, r_temp_1);
2112         if (out_regs2[out].first()->is_valid()) {
2113           double_move(masm, in_regs[in], out_regs2[out], r_callers_sp, r_temp_1);
2114         }
2115         break;
2116       case T_ADDRESS:
2117         fatal("found type (T_ADDRESS) in java args");
2118         break;
2119       default:
2120         ShouldNotReachHere();
2121         break;
2122     }
2123   }
2124 
2125   // Pre-load a static method's oop into ARG2.
2126   // Used both by locking code and the normal JNI call code.
2127   if (method_is_static && !is_critical_native) {
2128     __ set_oop_constant(JNIHandles::make_local(method->method_holder()->java_mirror()),
2129                         r_carg2_classorobject);
2130 
2131     // Now handlize the static class mirror in carg2. It's known not-null.
2132     __ std(r_carg2_classorobject, klass_offset, R1_SP);
2133     oop_map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
2134     __ addi(r_carg2_classorobject, R1_SP, klass_offset);
2135   }
2136 
2137   // Get JNIEnv* which is first argument to native.
2138   if (!is_critical_native) {
2139     __ addi(r_carg1_jnienv, R16_thread, in_bytes(JavaThread::jni_environment_offset()));
2140   }
2141 
2142   // NOTE:
2143   //
2144   // We have all of the arguments setup at this point.
2145   // We MUST NOT touch any outgoing regs from this point on.
2146   // So if we must call out we must push a new frame.
2147 
2148   // Get current pc for oopmap, and load it patchable relative to global toc.
2149   oopmap_pc = (intptr_t) __ pc();
2150   __ calculate_address_from_global_toc(r_return_pc, (address)oopmap_pc, true, true, true, true);
2151 
2152   // We use the same pc/oopMap repeatedly when we call out.
2153   oop_maps->add_gc_map(oopmap_pc - start_pc, oop_map);
2154 
2155   // r_return_pc now has the pc loaded that we will use when we finally call
2156   // to native.
2157 
2158   // Make sure that thread is non-volatile; it crosses a bunch of VM calls below.
2159   assert(R16_thread->is_nonvolatile(), "thread must be in non-volatile register");
2160 
2161 # if 0
2162   // DTrace method entry
2163 # endif
2164 
2165   // Lock a synchronized method.
2166   // --------------------------------------------------------------------------
2167 
2168   if (method->is_synchronized()) {
2169     assert(!is_critical_native, "unhandled");
2170     ConditionRegister r_flag = CCR1;
2171     Register          r_oop  = r_temp_4;
2172     const Register    r_box  = r_temp_5;
2173     Label             done, locked;
2174 
2175     // Load the oop for the object or class. r_carg2_classorobject contains
2176     // either the handlized oop from the incoming arguments or the handlized
2177     // class mirror (if the method is static).
2178     __ ld(r_oop, 0, r_carg2_classorobject);
2179 
2180     // Get the lock box slot's address.
2181     __ addi(r_box, R1_SP, lock_offset);
2182 
2183 #   ifdef ASSERT
2184     if (UseBiasedLocking) {
2185       // Making the box point to itself will make it clear it went unused
2186       // but also be obviously invalid.
2187       __ std(r_box, 0, r_box);
2188     }
2189 #   endif // ASSERT
2190 
2191     // Try fastpath for locking.
2192     // fast_lock kills r_temp_1, r_temp_2, r_temp_3.
2193     __ compiler_fast_lock_object(r_flag, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3);
2194     __ beq(r_flag, locked);
2195 
2196     // None of the above fast optimizations worked so we have to get into the
2197     // slow case of monitor enter. Inline a special case of call_VM that
2198     // disallows any pending_exception.
2199 
2200     // Save argument registers and leave room for C-compatible ABI_REG_ARGS.
2201     int frame_size = frame::abi_reg_args_size +
2202                      round_to(total_c_args * wordSize, frame::alignment_in_bytes);
2203     __ mr(R11_scratch1, R1_SP);
2204     RegisterSaver::push_frame_and_save_argument_registers(masm, R12_scratch2, frame_size, total_c_args, out_regs, out_regs2);
2205 
2206     // Do the call.
2207     __ set_last_Java_frame(R11_scratch1, r_return_pc);
2208     assert(r_return_pc->is_nonvolatile(), "expecting return pc to be in non-volatile register");
2209     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), r_oop, r_box, R16_thread);
2210     __ reset_last_Java_frame();
2211 
2212     RegisterSaver::restore_argument_registers_and_pop_frame(masm, frame_size, total_c_args, out_regs, out_regs2);
2213 
2214     __ asm_assert_mem8_is_zero(thread_(pending_exception),
2215        "no pending exception allowed on exit from SharedRuntime::complete_monitor_locking_C", 0);
2216 
2217     __ bind(locked);
2218   }
2219 
2220 
2221   // Publish thread state
2222   // --------------------------------------------------------------------------
2223 
2224   // Use that pc we placed in r_return_pc a while back as the current frame anchor.
2225   __ set_last_Java_frame(R1_SP, r_return_pc);
2226 
2227   // Transition from _thread_in_Java to _thread_in_native.
2228   __ li(R0, _thread_in_native);
2229   __ release();
2230   // TODO: PPC port assert(4 == JavaThread::sz_thread_state(), "unexpected field size");
2231   __ stw(R0, thread_(thread_state));
2232   if (UseMembar) {
2233     __ fence();
2234   }
2235 
2236 
2237   // The JNI call
2238   // --------------------------------------------------------------------------
2239 #if defined(ABI_ELFv2)
2240   __ call_c(native_func, relocInfo::runtime_call_type);
2241 #else
2242   FunctionDescriptor* fd_native_method = (FunctionDescriptor*) native_func;
2243   __ call_c(fd_native_method, relocInfo::runtime_call_type);
2244 #endif
2245 
2246 
2247   // Now, we are back from the native code.
2248 
2249 
2250   // Unpack the native result.
2251   // --------------------------------------------------------------------------
2252 
2253   // For int-types, we do any needed sign-extension required.
2254   // Care must be taken that the return values (R3_RET and F1_RET)
2255   // will survive any VM calls for blocking or unlocking.
2256   // An OOP result (handle) is done specially in the slow-path code.
2257 
2258   switch (ret_type) {
2259     case T_VOID:    break;        // Nothing to do!
2260     case T_FLOAT:   break;        // Got it where we want it (unless slow-path).
2261     case T_DOUBLE:  break;        // Got it where we want it (unless slow-path).
2262     case T_LONG:    break;        // Got it where we want it (unless slow-path).
2263     case T_OBJECT:  break;        // Really a handle.
2264                                   // Cannot de-handlize until after reclaiming jvm_lock.
2265     case T_ARRAY:   break;
2266 
2267     case T_BOOLEAN: {             // 0 -> false(0); !0 -> true(1)
2268       Label skip_modify;
2269       __ cmpwi(CCR0, R3_RET, 0);
2270       __ beq(CCR0, skip_modify);
2271       __ li(R3_RET, 1);
2272       __ bind(skip_modify);
2273       break;
2274       }
2275     case T_BYTE: {                // sign extension
2276       __ extsb(R3_RET, R3_RET);
2277       break;
2278       }
2279     case T_CHAR: {                // unsigned result
2280       __ andi(R3_RET, R3_RET, 0xffff);
2281       break;
2282       }
2283     case T_SHORT: {               // sign extension
2284       __ extsh(R3_RET, R3_RET);
2285       break;
2286       }
2287     case T_INT:                   // nothing to do
2288       break;
2289     default:
2290       ShouldNotReachHere();
2291       break;
2292   }
2293 
2294 
2295   // Publish thread state
2296   // --------------------------------------------------------------------------
2297 
2298   // Switch thread to "native transition" state before reading the
2299   // synchronization state. This additional state is necessary because reading
2300   // and testing the synchronization state is not atomic w.r.t. GC, as this
2301   // scenario demonstrates:
2302   //   - Java thread A, in _thread_in_native state, loads _not_synchronized
2303   //     and is preempted.
2304   //   - VM thread changes sync state to synchronizing and suspends threads
2305   //     for GC.
2306   //   - Thread A is resumed to finish this native method, but doesn't block
2307   //     here since it didn't see any synchronization in progress, and escapes.
2308 
2309   // Transition from _thread_in_native to _thread_in_native_trans.
2310   __ li(R0, _thread_in_native_trans);
2311   __ release();
2312   // TODO: PPC port assert(4 == JavaThread::sz_thread_state(), "unexpected field size");
2313   __ stw(R0, thread_(thread_state));
2314 
2315 
2316   // Must we block?
2317   // --------------------------------------------------------------------------
2318 
2319   // Block, if necessary, before resuming in _thread_in_Java state.
2320   // In order for GC to work, don't clear the last_Java_sp until after blocking.
2321   Label after_transition;
2322   {
2323     Label no_block, sync;
2324 
2325     if (os::is_MP()) {
2326       if (UseMembar) {
2327         // Force this write out before the read below.
2328         __ fence();
2329       } else {
2330         // Write serialization page so VM thread can do a pseudo remote membar.
2331         // We use the current thread pointer to calculate a thread specific
2332         // offset to write to within the page. This minimizes bus traffic
2333         // due to cache line collision.
2334         __ serialize_memory(R16_thread, r_temp_4, r_temp_5);
2335       }
2336     }
2337 
2338     Register sync_state_addr = r_temp_4;
2339     Register sync_state      = r_temp_5;
2340     Register suspend_flags   = r_temp_6;
2341 
2342     __ load_const(sync_state_addr, SafepointSynchronize::address_of_state(), /*temp*/ sync_state);
2343 
2344     // TODO: PPC port assert(4 == SafepointSynchronize::sz_state(), "unexpected field size");
2345     __ lwz(sync_state, 0, sync_state_addr);
2346 
2347     // TODO: PPC port assert(4 == Thread::sz_suspend_flags(), "unexpected field size");
2348     __ lwz(suspend_flags, thread_(suspend_flags));
2349 
2350     __ acquire();
2351 
2352     Label do_safepoint;
2353     // No synchronization in progress nor yet synchronized.
2354     __ cmpwi(CCR0, sync_state, SafepointSynchronize::_not_synchronized);
2355     // Not suspended.
2356     __ cmpwi(CCR1, suspend_flags, 0);
2357 
2358     __ bne(CCR0, sync);
2359     __ beq(CCR1, no_block);
2360 
2361     // Block. Save any potential method result value before the operation and
2362     // use a leaf call to leave the last_Java_frame setup undisturbed. Doing this
2363     // lets us share the oopMap we used when we went native rather than create
2364     // a distinct one for this pc.
2365     __ bind(sync);
2366 
2367     address entry_point = is_critical_native
2368       ? CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans_and_transition)
2369       : CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans);
2370     save_native_result(masm, ret_type, workspace_slot_offset);
2371     __ call_VM_leaf(entry_point, R16_thread);
2372     restore_native_result(masm, ret_type, workspace_slot_offset);
2373 
2374     if (is_critical_native) {
2375       __ b(after_transition); // No thread state transition here.
2376     }
2377     __ bind(no_block);
2378   }
2379 
2380   // Publish thread state.
2381   // --------------------------------------------------------------------------
2382 
2383   // Thread state is thread_in_native_trans. Any safepoint blocking has
2384   // already happened so we can now change state to _thread_in_Java.
2385 
2386   // Transition from _thread_in_native_trans to _thread_in_Java.
2387   __ li(R0, _thread_in_Java);
2388   __ release();
2389   // TODO: PPC port assert(4 == JavaThread::sz_thread_state(), "unexpected field size");
2390   __ stw(R0, thread_(thread_state));
2391   if (UseMembar) {
2392     __ fence();
2393   }
2394   __ bind(after_transition);
2395 
2396   // Reguard any pages if necessary.
2397   // --------------------------------------------------------------------------
2398 
2399   Label no_reguard;
2400   __ lwz(r_temp_1, thread_(stack_guard_state));
2401   __ cmpwi(CCR0, r_temp_1, JavaThread::stack_guard_yellow_disabled);
2402   __ bne(CCR0, no_reguard);
2403 
2404   save_native_result(masm, ret_type, workspace_slot_offset);
2405   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
2406   restore_native_result(masm, ret_type, workspace_slot_offset);
2407 
2408   __ bind(no_reguard);
2409 
2410 
2411   // Unlock
2412   // --------------------------------------------------------------------------
2413 
2414   if (method->is_synchronized()) {
2415 
2416     ConditionRegister r_flag   = CCR1;
2417     const Register r_oop       = r_temp_4;
2418     const Register r_box       = r_temp_5;
2419     const Register r_exception = r_temp_6;
2420     Label done;
2421 
2422     // Get oop and address of lock object box.
2423     if (method_is_static) {
2424       assert(klass_offset != -1, "");
2425       __ ld(r_oop, klass_offset, R1_SP);
2426     } else {
2427       assert(receiver_offset != -1, "");
2428       __ ld(r_oop, receiver_offset, R1_SP);
2429     }
2430     __ addi(r_box, R1_SP, lock_offset);
2431 
2432     // Try fastpath for unlocking.
2433     __ compiler_fast_unlock_object(r_flag, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3);
2434     __ beq(r_flag, done);
2435 
2436     // Save and restore any potential method result value around the unlocking operation.
2437     save_native_result(masm, ret_type, workspace_slot_offset);
2438 
2439     // Must save pending exception around the slow-path VM call. Since it's a
2440     // leaf call, the pending exception (if any) can be kept in a register.
2441     __ ld(r_exception, thread_(pending_exception));
2442     assert(r_exception->is_nonvolatile(), "exception register must be non-volatile");
2443     __ li(R0, 0);
2444     __ std(R0, thread_(pending_exception));
2445 
2446     // Slow case of monitor enter.
2447     // Inline a special case of call_VM that disallows any pending_exception.
2448     // Arguments are (oop obj, BasicLock* lock, JavaThread* thread).
2449     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), r_oop, r_box, R16_thread);
2450 
2451     __ asm_assert_mem8_is_zero(thread_(pending_exception),
2452        "no pending exception allowed on exit from SharedRuntime::complete_monitor_unlocking_C", 0);
2453 
2454     restore_native_result(masm, ret_type, workspace_slot_offset);
2455 
2456     // Check_forward_pending_exception jump to forward_exception if any pending
2457     // exception is set. The forward_exception routine expects to see the
2458     // exception in pending_exception and not in a register. Kind of clumsy,
2459     // since all folks who branch to forward_exception must have tested
2460     // pending_exception first and hence have it in a register already.
2461     __ std(r_exception, thread_(pending_exception));
2462 
2463     __ bind(done);
2464   }
2465 
2466 # if 0
2467   // DTrace method exit
2468 # endif
2469 
2470   // Clear "last Java frame" SP and PC.
2471   // --------------------------------------------------------------------------
2472 
2473   __ reset_last_Java_frame();
2474 
2475   // Unpack oop result.
2476   // --------------------------------------------------------------------------
2477 
2478   if (ret_type == T_OBJECT || ret_type == T_ARRAY) {
2479     Label skip_unboxing;
2480     __ cmpdi(CCR0, R3_RET, 0);
2481     __ beq(CCR0, skip_unboxing);
2482     __ ld(R3_RET, 0, R3_RET);
2483     __ bind(skip_unboxing);
2484     __ verify_oop(R3_RET);
2485   }
2486 
2487 
2488   // Reset handle block.
2489   // --------------------------------------------------------------------------
2490   if (!is_critical_native) {
2491   __ ld(r_temp_1, thread_(active_handles));
2492   // TODO: PPC port assert(4 == JNIHandleBlock::top_size_in_bytes(), "unexpected field size");
2493   __ li(r_temp_2, 0);
2494   __ stw(r_temp_2, JNIHandleBlock::top_offset_in_bytes(), r_temp_1);
2495 
2496 
2497   // Check for pending exceptions.
2498   // --------------------------------------------------------------------------
2499   __ ld(r_temp_2, thread_(pending_exception));
2500   __ cmpdi(CCR0, r_temp_2, 0);
2501   __ bne(CCR0, handle_pending_exception);
2502   }
2503 
2504   // Return
2505   // --------------------------------------------------------------------------
2506 
2507   __ pop_frame();
2508   __ restore_LR_CR(R11);
2509   __ blr();
2510 
2511 
2512   // Handler for pending exceptions (out-of-line).
2513   // --------------------------------------------------------------------------
2514 
2515   // Since this is a native call, we know the proper exception handler
2516   // is the empty function. We just pop this frame and then jump to
2517   // forward_exception_entry.
2518   if (!is_critical_native) {
2519   __ align(InteriorEntryAlignment);
2520   __ bind(handle_pending_exception);
2521 
2522   __ pop_frame();
2523   __ restore_LR_CR(R11);
2524   __ b64_patchable((address)StubRoutines::forward_exception_entry(),
2525                        relocInfo::runtime_call_type);
2526   }
2527 
2528   // Handler for a cache miss (out-of-line).
2529   // --------------------------------------------------------------------------
2530 
2531   if (!method_is_static) {
2532   __ align(InteriorEntryAlignment);
2533   __ bind(ic_miss);
2534 
2535   __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
2536                        relocInfo::runtime_call_type);
2537   }
2538 
2539   // Done.
2540   // --------------------------------------------------------------------------
2541 
2542   __ flush();
2543 
2544   nmethod *nm = nmethod::new_native_nmethod(method,
2545                                             compile_id,
2546                                             masm->code(),
2547                                             vep_start_pc-start_pc,
2548                                             frame_done_pc-start_pc,
2549                                             stack_slots / VMRegImpl::slots_per_word,
2550                                             (method_is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2551                                             in_ByteSize(lock_offset),
2552                                             oop_maps);
2553 
2554   if (is_critical_native) {
2555     nm->set_lazy_critical_native(true);
2556   }
2557 
2558   return nm;
2559 #else
2560   ShouldNotReachHere();
2561   return NULL;
2562 #endif // COMPILER2
2563 }
2564 
2565 // This function returns the adjust size (in number of words) to a c2i adapter
2566 // activation for use during deoptimization.
2567 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) {
2568   return round_to((callee_locals - callee_parameters) * Interpreter::stackElementWords, frame::alignment_in_bytes);
2569 }
2570 
2571 uint SharedRuntime::out_preserve_stack_slots() {
2572 #if defined(COMPILER1) || defined(COMPILER2)
2573   return frame::jit_out_preserve_size / VMRegImpl::stack_slot_size;
2574 #else
2575   return 0;
2576 #endif
2577 }
2578 
2579 #ifdef COMPILER2
2580 // Frame generation for deopt and uncommon trap blobs.
2581 static void push_skeleton_frame(MacroAssembler* masm, bool deopt,
2582                                 /* Read */
2583                                 Register unroll_block_reg,
2584                                 /* Update */
2585                                 Register frame_sizes_reg,
2586                                 Register number_of_frames_reg,
2587                                 Register pcs_reg,
2588                                 /* Invalidate */
2589                                 Register frame_size_reg,
2590                                 Register pc_reg) {
2591 
2592   __ ld(pc_reg, 0, pcs_reg);
2593   __ ld(frame_size_reg, 0, frame_sizes_reg);
2594   __ std(pc_reg, _abi(lr), R1_SP);
2595   __ push_frame(frame_size_reg, R0/*tmp*/);
2596 #ifdef CC_INTERP
2597   __ std(R1_SP, _parent_ijava_frame_abi(initial_caller_sp), R1_SP);
2598 #else
2599 #ifdef ASSERT
2600   __ load_const_optimized(pc_reg, 0x5afe);
2601   __ std(pc_reg, _ijava_state_neg(ijava_reserved), R1_SP);
2602 #endif
2603   __ std(R1_SP, _ijava_state_neg(sender_sp), R1_SP);
2604 #endif // CC_INTERP
2605   __ addi(number_of_frames_reg, number_of_frames_reg, -1);
2606   __ addi(frame_sizes_reg, frame_sizes_reg, wordSize);
2607   __ addi(pcs_reg, pcs_reg, wordSize);
2608 }
2609 
2610 // Loop through the UnrollBlock info and create new frames.
2611 static void push_skeleton_frames(MacroAssembler* masm, bool deopt,
2612                                  /* read */
2613                                  Register unroll_block_reg,
2614                                  /* invalidate */
2615                                  Register frame_sizes_reg,
2616                                  Register number_of_frames_reg,
2617                                  Register pcs_reg,
2618                                  Register frame_size_reg,
2619                                  Register pc_reg) {
2620   Label loop;
2621 
2622  // _number_of_frames is of type int (deoptimization.hpp)
2623   __ lwa(number_of_frames_reg,
2624              Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes(),
2625              unroll_block_reg);
2626   __ ld(pcs_reg,
2627             Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes(),
2628             unroll_block_reg);
2629   __ ld(frame_sizes_reg,
2630             Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes(),
2631             unroll_block_reg);
2632 
2633   // stack: (caller_of_deoptee, ...).
2634 
2635   // At this point we either have an interpreter frame or a compiled
2636   // frame on top of stack. If it is a compiled frame we push a new c2i
2637   // adapter here
2638 
2639   // Memorize top-frame stack-pointer.
2640   __ mr(frame_size_reg/*old_sp*/, R1_SP);
2641 
2642   // Resize interpreter top frame OR C2I adapter.
2643 
2644   // At this moment, the top frame (which is the caller of the deoptee) is
2645   // an interpreter frame or a newly pushed C2I adapter or an entry frame.
2646   // The top frame has a TOP_IJAVA_FRAME_ABI and the frame contains the
2647   // outgoing arguments.
2648   //
2649   // In order to push the interpreter frame for the deoptee, we need to
2650   // resize the top frame such that we are able to place the deoptee's
2651   // locals in the frame.
2652   // Additionally, we have to turn the top frame's TOP_IJAVA_FRAME_ABI
2653   // into a valid PARENT_IJAVA_FRAME_ABI.
2654 
2655   __ lwa(R11_scratch1,
2656              Deoptimization::UnrollBlock::caller_adjustment_offset_in_bytes(),
2657              unroll_block_reg);
2658   __ neg(R11_scratch1, R11_scratch1);
2659 
2660   // R11_scratch1 contains size of locals for frame resizing.
2661   // R12_scratch2 contains top frame's lr.
2662 
2663   // Resize frame by complete frame size prevents TOC from being
2664   // overwritten by locals. A more stack space saving way would be
2665   // to copy the TOC to its location in the new abi.
2666   __ addi(R11_scratch1, R11_scratch1, - frame::parent_ijava_frame_abi_size);
2667 
2668   // now, resize the frame
2669   __ resize_frame(R11_scratch1, pc_reg/*tmp*/);
2670 
2671   // In the case where we have resized a c2i frame above, the optional
2672   // alignment below the locals has size 32 (why?).
2673   __ std(R12_scratch2, _abi(lr), R1_SP);
2674 
2675   // Initialize initial_caller_sp.
2676 #ifdef CC_INTERP
2677   __ std(frame_size_reg/*old_sp*/, _parent_ijava_frame_abi(initial_caller_sp), R1_SP);
2678 #else
2679 #ifdef ASSERT
2680  __ load_const_optimized(pc_reg, 0x5afe);
2681  __ std(pc_reg, _ijava_state_neg(ijava_reserved), R1_SP);
2682 #endif
2683  __ std(frame_size_reg, _ijava_state_neg(sender_sp), R1_SP);
2684 #endif // CC_INTERP
2685 
2686 #ifdef ASSERT
2687   // Make sure that there is at least one entry in the array.
2688   __ cmpdi(CCR0, number_of_frames_reg, 0);
2689   __ asm_assert_ne("array_size must be > 0", 0x205);
2690 #endif
2691 
2692   // Now push the new interpreter frames.
2693   //
2694   __ bind(loop);
2695   // Allocate a new frame, fill in the pc.
2696   push_skeleton_frame(masm, deopt,
2697                       unroll_block_reg,
2698                       frame_sizes_reg,
2699                       number_of_frames_reg,
2700                       pcs_reg,
2701                       frame_size_reg,
2702                       pc_reg);
2703   __ cmpdi(CCR0, number_of_frames_reg, 0);
2704   __ bne(CCR0, loop);
2705 
2706   // Get the return address pointing into the frame manager.
2707   __ ld(R0, 0, pcs_reg);
2708   // Store it in the top interpreter frame.
2709   __ std(R0, _abi(lr), R1_SP);
2710   // Initialize frame_manager_lr of interpreter top frame.
2711 #ifdef CC_INTERP
2712   __ std(R0, _top_ijava_frame_abi(frame_manager_lr), R1_SP);
2713 #endif
2714 }
2715 #endif
2716 
2717 void SharedRuntime::generate_deopt_blob() {
2718   // Allocate space for the code
2719   ResourceMark rm;
2720   // Setup code generation tools
2721   CodeBuffer buffer("deopt_blob", 2048, 1024);
2722   InterpreterMacroAssembler* masm = new InterpreterMacroAssembler(&buffer);
2723   Label exec_mode_initialized;
2724   int frame_size_in_words;
2725   OopMap* map = NULL;
2726   OopMapSet *oop_maps = new OopMapSet();
2727 
2728   // size of ABI112 plus spill slots for R3_RET and F1_RET.
2729   const int frame_size_in_bytes = frame::abi_reg_args_spill_size;
2730   const int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
2731   int first_frame_size_in_bytes = 0; // frame size of "unpack frame" for call to fetch_unroll_info.
2732 
2733   const Register exec_mode_reg = R21_tmp1;
2734 
2735   const address start = __ pc();
2736 
2737 #ifdef COMPILER2
2738   // --------------------------------------------------------------------------
2739   // Prolog for non exception case!
2740 
2741   // We have been called from the deopt handler of the deoptee.
2742   //
2743   // deoptee:
2744   //                      ...
2745   //                      call X
2746   //                      ...
2747   //  deopt_handler:      call_deopt_stub
2748   //  cur. return pc  --> ...
2749   //
2750   // So currently SR_LR points behind the call in the deopt handler.
2751   // We adjust it such that it points to the start of the deopt handler.
2752   // The return_pc has been stored in the frame of the deoptee and
2753   // will replace the address of the deopt_handler in the call
2754   // to Deoptimization::fetch_unroll_info below.
2755   // We can't grab a free register here, because all registers may
2756   // contain live values, so let the RegisterSaver do the adjustment
2757   // of the return pc.
2758   const int return_pc_adjustment_no_exception = -HandlerImpl::size_deopt_handler();
2759 
2760   // Push the "unpack frame"
2761   // Save everything in sight.
2762   map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
2763                                                                    &first_frame_size_in_bytes,
2764                                                                    /*generate_oop_map=*/ true,
2765                                                                    return_pc_adjustment_no_exception,
2766                                                                    RegisterSaver::return_pc_is_lr);
2767   assert(map != NULL, "OopMap must have been created");
2768 
2769   __ li(exec_mode_reg, Deoptimization::Unpack_deopt);
2770   // Save exec mode for unpack_frames.
2771   __ b(exec_mode_initialized);
2772 
2773   // --------------------------------------------------------------------------
2774   // Prolog for exception case
2775 
2776   // An exception is pending.
2777   // We have been called with a return (interpreter) or a jump (exception blob).
2778   //
2779   // - R3_ARG1: exception oop
2780   // - R4_ARG2: exception pc
2781 
2782   int exception_offset = __ pc() - start;
2783 
2784   BLOCK_COMMENT("Prolog for exception case");
2785 
2786   // The RegisterSaves doesn't need to adjust the return pc for this situation.
2787   const int return_pc_adjustment_exception = 0;
2788 
2789   // Push the "unpack frame".
2790   // Save everything in sight.
2791   assert(R4 == R4_ARG2, "exception pc must be in r4");
2792   RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
2793                                                              &first_frame_size_in_bytes,
2794                                                              /*generate_oop_map=*/ false,
2795                                                              return_pc_adjustment_exception,
2796                                                              RegisterSaver::return_pc_is_r4);
2797 
2798   // Deopt during an exception. Save exec mode for unpack_frames.
2799   __ li(exec_mode_reg, Deoptimization::Unpack_exception);
2800 
2801   // Store exception oop and pc in thread (location known to GC).
2802   // This is needed since the call to "fetch_unroll_info()" may safepoint.
2803   __ std(R3_ARG1, in_bytes(JavaThread::exception_oop_offset()), R16_thread);
2804   __ std(R4_ARG2, in_bytes(JavaThread::exception_pc_offset()),  R16_thread);
2805 
2806   // fall through
2807 
2808   // --------------------------------------------------------------------------
2809   __ BIND(exec_mode_initialized);
2810 
2811   {
2812   const Register unroll_block_reg = R22_tmp2;
2813 
2814   // We need to set `last_Java_frame' because `fetch_unroll_info' will
2815   // call `last_Java_frame()'. The value of the pc in the frame is not
2816   // particularly important. It just needs to identify this blob.
2817   __ set_last_Java_frame(R1_SP, noreg);
2818 
2819   // With EscapeAnalysis turned on, this call may safepoint!
2820   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info), R16_thread, exec_mode_reg);
2821   address calls_return_pc = __ last_calls_return_pc();
2822   // Set an oopmap for the call site that describes all our saved registers.
2823   oop_maps->add_gc_map(calls_return_pc - start, map);
2824 
2825   __ reset_last_Java_frame();
2826   // Save the return value.
2827   __ mr(unroll_block_reg, R3_RET);
2828 
2829   // Restore only the result registers that have been saved
2830   // by save_volatile_registers(...).
2831   RegisterSaver::restore_result_registers(masm, first_frame_size_in_bytes);
2832 
2833   // reload the exec mode from the UnrollBlock (it might have changed)
2834   __ lwz(exec_mode_reg, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes(), unroll_block_reg);
2835   // In excp_deopt_mode, restore and clear exception oop which we
2836   // stored in the thread during exception entry above. The exception
2837   // oop will be the return value of this stub.
2838   Label skip_restore_excp;
2839   __ cmpdi(CCR0, exec_mode_reg, Deoptimization::Unpack_exception);
2840   __ bne(CCR0, skip_restore_excp);
2841   __ ld(R3_RET, in_bytes(JavaThread::exception_oop_offset()), R16_thread);
2842   __ ld(R4_ARG2, in_bytes(JavaThread::exception_pc_offset()), R16_thread);
2843   __ li(R0, 0);
2844   __ std(R0, in_bytes(JavaThread::exception_pc_offset()),  R16_thread);
2845   __ std(R0, in_bytes(JavaThread::exception_oop_offset()), R16_thread);
2846   __ BIND(skip_restore_excp);
2847 
2848   __ pop_frame();
2849 
2850   // stack: (deoptee, optional i2c, caller of deoptee, ...).
2851 
2852   // pop the deoptee's frame
2853   __ pop_frame();
2854 
2855   // stack: (caller_of_deoptee, ...).
2856 
2857   // Loop through the `UnrollBlock' info and create interpreter frames.
2858   push_skeleton_frames(masm, true/*deopt*/,
2859                        unroll_block_reg,
2860                        R23_tmp3,
2861                        R24_tmp4,
2862                        R25_tmp5,
2863                        R26_tmp6,
2864                        R27_tmp7);
2865 
2866   // stack: (skeletal interpreter frame, ..., optional skeletal
2867   // interpreter frame, optional c2i, caller of deoptee, ...).
2868   }
2869 
2870   // push an `unpack_frame' taking care of float / int return values.
2871   __ push_frame(frame_size_in_bytes, R0/*tmp*/);
2872 
2873   // stack: (unpack frame, skeletal interpreter frame, ..., optional
2874   // skeletal interpreter frame, optional c2i, caller of deoptee,
2875   // ...).
2876 
2877   // Spill live volatile registers since we'll do a call.
2878   __ std( R3_RET, _abi_reg_args_spill(spill_ret),  R1_SP);
2879   __ stfd(F1_RET, _abi_reg_args_spill(spill_fret), R1_SP);
2880 
2881   // Let the unpacker layout information in the skeletal frames just
2882   // allocated.
2883   __ get_PC_trash_LR(R3_RET);
2884   __ set_last_Java_frame(/*sp*/R1_SP, /*pc*/R3_RET);
2885   // This is a call to a LEAF method, so no oop map is required.
2886   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames),
2887                   R16_thread/*thread*/, exec_mode_reg/*exec_mode*/);
2888   __ reset_last_Java_frame();
2889 
2890   // Restore the volatiles saved above.
2891   __ ld( R3_RET, _abi_reg_args_spill(spill_ret),  R1_SP);
2892   __ lfd(F1_RET, _abi_reg_args_spill(spill_fret), R1_SP);
2893 
2894   // Pop the unpack frame.
2895   __ pop_frame();
2896   __ restore_LR_CR(R0);
2897 
2898   // stack: (top interpreter frame, ..., optional interpreter frame,
2899   // optional c2i, caller of deoptee, ...).
2900 
2901   // Initialize R14_state.
2902 #ifdef CC_INTERP
2903   __ ld(R14_state, 0, R1_SP);
2904   __ addi(R14_state, R14_state, -frame::interpreter_frame_cinterpreterstate_size_in_bytes());
2905   // Also inititialize R15_prev_state.
2906   __ restore_prev_state();
2907 #else
2908   __ restore_interpreter_state(R11_scratch1);
2909   __ load_const_optimized(R25_templateTableBase, (address)Interpreter::dispatch_table((TosState)0), R11_scratch1);
2910 #endif // CC_INTERP
2911 
2912 
2913   // Return to the interpreter entry point.
2914   __ blr();
2915   __ flush();
2916 #else // COMPILER2
2917   __ unimplemented("deopt blob needed only with compiler");
2918   int exception_offset = __ pc() - start;
2919 #endif // COMPILER2
2920 
2921   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, 0, first_frame_size_in_bytes / wordSize);
2922 }
2923 
2924 #ifdef COMPILER2
2925 void SharedRuntime::generate_uncommon_trap_blob() {
2926   // Allocate space for the code.
2927   ResourceMark rm;
2928   // Setup code generation tools.
2929   CodeBuffer buffer("uncommon_trap_blob", 2048, 1024);
2930   InterpreterMacroAssembler* masm = new InterpreterMacroAssembler(&buffer);
2931   address start = __ pc();
2932 
2933   Register unroll_block_reg = R21_tmp1;
2934   Register klass_index_reg  = R22_tmp2;
2935   Register unc_trap_reg     = R23_tmp3;
2936 
2937   OopMapSet* oop_maps = new OopMapSet();
2938   int frame_size_in_bytes = frame::abi_reg_args_size;
2939   OopMap* map = new OopMap(frame_size_in_bytes / sizeof(jint), 0);
2940 
2941   // stack: (deoptee, optional i2c, caller_of_deoptee, ...).
2942 
2943   // Push a dummy `unpack_frame' and call
2944   // `Deoptimization::uncommon_trap' to pack the compiled frame into a
2945   // vframe array and return the `UnrollBlock' information.
2946 
2947   // Save LR to compiled frame.
2948   __ save_LR_CR(R11_scratch1);
2949 
2950   // Push an "uncommon_trap" frame.
2951   __ push_frame_reg_args(0, R11_scratch1);
2952 
2953   // stack: (unpack frame, deoptee, optional i2c, caller_of_deoptee, ...).
2954 
2955   // Set the `unpack_frame' as last_Java_frame.
2956   // `Deoptimization::uncommon_trap' expects it and considers its
2957   // sender frame as the deoptee frame.
2958   // Remember the offset of the instruction whose address will be
2959   // moved to R11_scratch1.
2960   address gc_map_pc = __ get_PC_trash_LR(R11_scratch1);
2961 
2962   __ set_last_Java_frame(/*sp*/R1_SP, /*pc*/R11_scratch1);
2963 
2964   __ mr(klass_index_reg, R3);
2965   __ li(R5, Deoptimization::Unpack_exception);
2966   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap),
2967                   R16_thread, klass_index_reg, R5);
2968 
2969   // Set an oopmap for the call site.
2970   oop_maps->add_gc_map(gc_map_pc - start, map);
2971 
2972   __ reset_last_Java_frame();
2973 
2974   // Pop the `unpack frame'.
2975   __ pop_frame();
2976 
2977   // stack: (deoptee, optional i2c, caller_of_deoptee, ...).
2978 
2979   // Save the return value.
2980   __ mr(unroll_block_reg, R3_RET);
2981 
2982   // Pop the uncommon_trap frame.
2983   __ pop_frame();
2984 
2985   // stack: (caller_of_deoptee, ...).
2986 
2987 #ifdef ASSERT
2988   __ lwz(R22_tmp2, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes(), unroll_block_reg);
2989   __ cmpdi(CCR0, R22_tmp2, (unsigned)Deoptimization::Unpack_uncommon_trap);
2990   __ asm_assert_eq("SharedRuntime::generate_deopt_blob: expected Unpack_uncommon_trap", 0);
2991 #endif
2992 
2993   // Allocate new interpreter frame(s) and possibly a c2i adapter
2994   // frame.
2995   push_skeleton_frames(masm, false/*deopt*/,
2996                        unroll_block_reg,
2997                        R22_tmp2,
2998                        R23_tmp3,
2999                        R24_tmp4,
3000                        R25_tmp5,
3001                        R26_tmp6);
3002 
3003   // stack: (skeletal interpreter frame, ..., optional skeletal
3004   // interpreter frame, optional c2i, caller of deoptee, ...).
3005 
3006   // Push a dummy `unpack_frame' taking care of float return values.
3007   // Call `Deoptimization::unpack_frames' to layout information in the
3008   // interpreter frames just created.
3009 
3010   // Push a simple "unpack frame" here.
3011   __ push_frame_reg_args(0, R11_scratch1);
3012 
3013   // stack: (unpack frame, skeletal interpreter frame, ..., optional
3014   // skeletal interpreter frame, optional c2i, caller of deoptee,
3015   // ...).
3016 
3017   // Set the "unpack_frame" as last_Java_frame.
3018   __ get_PC_trash_LR(R11_scratch1);
3019   __ set_last_Java_frame(/*sp*/R1_SP, /*pc*/R11_scratch1);
3020 
3021   // Indicate it is the uncommon trap case.
3022   __ li(unc_trap_reg, Deoptimization::Unpack_uncommon_trap);
3023   // Let the unpacker layout information in the skeletal frames just
3024   // allocated.
3025   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames),
3026                   R16_thread, unc_trap_reg);
3027 
3028   __ reset_last_Java_frame();
3029   // Pop the `unpack frame'.
3030   __ pop_frame();
3031   // Restore LR from top interpreter frame.
3032   __ restore_LR_CR(R11_scratch1);
3033 
3034   // stack: (top interpreter frame, ..., optional interpreter frame,
3035   // optional c2i, caller of deoptee, ...).
3036 
3037 #ifdef CC_INTERP
3038   // Initialize R14_state, ...
3039   __ ld(R11_scratch1, 0, R1_SP);
3040   __ addi(R14_state, R11_scratch1, -frame::interpreter_frame_cinterpreterstate_size_in_bytes());
3041   // also initialize R15_prev_state.
3042   __ restore_prev_state();
3043 #else
3044   __ restore_interpreter_state(R11_scratch1);
3045   __ load_const_optimized(R25_templateTableBase, (address)Interpreter::dispatch_table((TosState)0), R11_scratch1);
3046 #endif // CC_INTERP
3047 
3048   // Return to the interpreter entry point.
3049   __ blr();
3050 
3051   masm->flush();
3052 
3053   _uncommon_trap_blob = UncommonTrapBlob::create(&buffer, oop_maps, frame_size_in_bytes/wordSize);
3054 }
3055 #endif // COMPILER2
3056 
3057 // Generate a special Compile2Runtime blob that saves all registers, and setup oopmap.
3058 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
3059   assert(StubRoutines::forward_exception_entry() != NULL,
3060          "must be generated before");
3061 
3062   ResourceMark rm;
3063   OopMapSet *oop_maps = new OopMapSet();
3064   OopMap* map;
3065 
3066   // Allocate space for the code. Setup code generation tools.
3067   CodeBuffer buffer("handler_blob", 2048, 1024);
3068   MacroAssembler* masm = new MacroAssembler(&buffer);
3069 
3070   address start = __ pc();
3071   int frame_size_in_bytes = 0;
3072 
3073   RegisterSaver::ReturnPCLocation return_pc_location;
3074   bool cause_return = (poll_type == POLL_AT_RETURN);
3075   if (cause_return) {
3076     // Nothing to do here. The frame has already been popped in MachEpilogNode.
3077     // Register LR already contains the return pc.
3078     return_pc_location = RegisterSaver::return_pc_is_lr;
3079   } else {
3080     // Use thread()->saved_exception_pc() as return pc.
3081     return_pc_location = RegisterSaver::return_pc_is_thread_saved_exception_pc;
3082   }
3083 
3084   // Save registers, fpu state, and flags.
3085   map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
3086                                                                    &frame_size_in_bytes,
3087                                                                    /*generate_oop_map=*/ true,
3088                                                                    /*return_pc_adjustment=*/0,
3089                                                                    return_pc_location);
3090 
3091   // The following is basically a call_VM. However, we need the precise
3092   // address of the call in order to generate an oopmap. Hence, we do all the
3093   // work outselves.
3094   __ set_last_Java_frame(/*sp=*/R1_SP, /*pc=*/noreg);
3095 
3096   // The return address must always be correct so that the frame constructor
3097   // never sees an invalid pc.
3098 
3099   // Do the call
3100   __ call_VM_leaf(call_ptr, R16_thread);
3101   address calls_return_pc = __ last_calls_return_pc();
3102 
3103   // Set an oopmap for the call site. This oopmap will map all
3104   // oop-registers and debug-info registers as callee-saved. This
3105   // will allow deoptimization at this safepoint to find all possible
3106   // debug-info recordings, as well as let GC find all oops.
3107   oop_maps->add_gc_map(calls_return_pc - start, map);
3108 
3109   Label noException;
3110 
3111   // Clear the last Java frame.
3112   __ reset_last_Java_frame();
3113 
3114   BLOCK_COMMENT("  Check pending exception.");
3115   const Register pending_exception = R0;
3116   __ ld(pending_exception, thread_(pending_exception));
3117   __ cmpdi(CCR0, pending_exception, 0);
3118   __ beq(CCR0, noException);
3119 
3120   // Exception pending
3121   RegisterSaver::restore_live_registers_and_pop_frame(masm,
3122                                                       frame_size_in_bytes,
3123                                                       /*restore_ctr=*/true);
3124 
3125   BLOCK_COMMENT("  Jump to forward_exception_entry.");
3126   // Jump to forward_exception_entry, with the issuing PC in LR
3127   // so it looks like the original nmethod called forward_exception_entry.
3128   __ b64_patchable(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
3129 
3130   // No exception case.
3131   __ BIND(noException);
3132 
3133 
3134   // Normal exit, restore registers and exit.
3135   RegisterSaver::restore_live_registers_and_pop_frame(masm,
3136                                                       frame_size_in_bytes,
3137                                                       /*restore_ctr=*/true);
3138 
3139   __ blr();
3140 
3141   // Make sure all code is generated
3142   masm->flush();
3143 
3144   // Fill-out other meta info
3145   // CodeBlob frame size is in words.
3146   return SafepointBlob::create(&buffer, oop_maps, frame_size_in_bytes / wordSize);
3147 }
3148 
3149 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss)
3150 //
3151 // Generate a stub that calls into the vm to find out the proper destination
3152 // of a java call. All the argument registers are live at this point
3153 // but since this is generic code we don't know what they are and the caller
3154 // must do any gc of the args.
3155 //
3156 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
3157 
3158   // allocate space for the code
3159   ResourceMark rm;
3160 
3161   CodeBuffer buffer(name, 1000, 512);
3162   MacroAssembler* masm = new MacroAssembler(&buffer);
3163 
3164   int frame_size_in_bytes;
3165 
3166   OopMapSet *oop_maps = new OopMapSet();
3167   OopMap* map = NULL;
3168 
3169   address start = __ pc();
3170 
3171   map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
3172                                                                    &frame_size_in_bytes,
3173                                                                    /*generate_oop_map*/ true,
3174                                                                    /*return_pc_adjustment*/ 0,
3175                                                                    RegisterSaver::return_pc_is_lr);
3176 
3177   // Use noreg as last_Java_pc, the return pc will be reconstructed
3178   // from the physical frame.
3179   __ set_last_Java_frame(/*sp*/R1_SP, noreg);
3180 
3181   int frame_complete = __ offset();
3182 
3183   // Pass R19_method as 2nd (optional) argument, used by
3184   // counter_overflow_stub.
3185   __ call_VM_leaf(destination, R16_thread, R19_method);
3186   address calls_return_pc = __ last_calls_return_pc();
3187   // Set an oopmap for the call site.
3188   // We need this not only for callee-saved registers, but also for volatile
3189   // registers that the compiler might be keeping live across a safepoint.
3190   // Create the oopmap for the call's return pc.
3191   oop_maps->add_gc_map(calls_return_pc - start, map);
3192 
3193   // R3_RET contains the address we are going to jump to assuming no exception got installed.
3194 
3195   // clear last_Java_sp
3196   __ reset_last_Java_frame();
3197 
3198   // Check for pending exceptions.
3199   BLOCK_COMMENT("Check for pending exceptions.");
3200   Label pending;
3201   __ ld(R11_scratch1, thread_(pending_exception));
3202   __ cmpdi(CCR0, R11_scratch1, 0);
3203   __ bne(CCR0, pending);
3204 
3205   __ mtctr(R3_RET); // Ctr will not be touched by restore_live_registers_and_pop_frame.
3206 
3207   RegisterSaver::restore_live_registers_and_pop_frame(masm, frame_size_in_bytes, /*restore_ctr*/ false);
3208 
3209   // Get the returned method.
3210   __ get_vm_result_2(R19_method);
3211 
3212   __ bctr();
3213 
3214 
3215   // Pending exception after the safepoint.
3216   __ BIND(pending);
3217 
3218   RegisterSaver::restore_live_registers_and_pop_frame(masm, frame_size_in_bytes, /*restore_ctr*/ true);
3219 
3220   // exception pending => remove activation and forward to exception handler
3221 
3222   __ li(R11_scratch1, 0);
3223   __ ld(R3_ARG1, thread_(pending_exception));
3224   __ std(R11_scratch1, in_bytes(JavaThread::vm_result_offset()), R16_thread);
3225   __ b64_patchable(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
3226 
3227   // -------------
3228   // Make sure all code is generated.
3229   masm->flush();
3230 
3231   // return the blob
3232   // frame_size_words or bytes??
3233   return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_bytes/wordSize,
3234                                        oop_maps, true);
3235 }