1 /*
   2  * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2016, 2017 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "interpreter/interpreterRuntime.hpp"
  30 #include "interpreter/interp_masm.hpp"
  31 #include "interpreter/templateTable.hpp"
  32 #include "memory/universe.hpp"
  33 #include "oops/objArrayKlass.hpp"
  34 #include "oops/oop.inline.hpp"
  35 #include "prims/methodHandles.hpp"
  36 #include "runtime/frame.inline.hpp"
  37 #include "runtime/safepointMechanism.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/synchronizer.hpp"
  41 
  42 #ifdef PRODUCT
  43 #define __ _masm->
  44 #define BLOCK_COMMENT(str)
  45 #define BIND(label)        __ bind(label);
  46 #else
  47 #define __ (PRODUCT_ONLY(false&&)Verbose ? (_masm->block_comment(FILE_AND_LINE),_masm):_masm)->
  48 #define BLOCK_COMMENT(str) __ block_comment(str)
  49 #define BIND(label)        __ bind(label); BLOCK_COMMENT(#label ":")
  50 #endif
  51 
  52 // The assumed minimum size of a BranchTableBlock.
  53 // The actual size of each block heavily depends on the CPU capabilities and,
  54 // of course, on the logic implemented in each block.
  55 #ifdef ASSERT
  56   #define BTB_MINSIZE 256
  57 #else
  58   #define BTB_MINSIZE  64
  59 #endif
  60 
  61 #ifdef ASSERT
  62 // Macro to open a BranchTableBlock (a piece of code that is branched to by a calculated branch).
  63 #define BTB_BEGIN(lbl, alignment, name)                                        \
  64   __ align_address(alignment);                                                 \
  65   __ bind(lbl);                                                                \
  66   { unsigned int b_off = __ offset();                                          \
  67     uintptr_t   b_addr = (uintptr_t)__ pc();                                   \
  68     __ z_larl(Z_R0, (int64_t)0);     /* Check current address alignment. */    \
  69     __ z_slgr(Z_R0, br_tab);         /* Current Address must be equal    */    \
  70     __ z_slgr(Z_R0, flags);          /* to calculated branch target.     */    \
  71     __ z_brc(Assembler::bcondLogZero, 3); /* skip trap if ok. */               \
  72     __ z_illtrap(0x55);                                                        \
  73     guarantee(b_addr%alignment == 0, "bad alignment at begin of block" name);
  74 
  75 // Macro to close a BranchTableBlock (a piece of code that is branched to by a calculated branch).
  76 #define BTB_END(lbl, alignment, name)                                          \
  77     uintptr_t   e_addr = (uintptr_t)__ pc();                                   \
  78     unsigned int e_off = __ offset();                                          \
  79     unsigned int len   = e_off-b_off;                                          \
  80     if (len > alignment) {                                                     \
  81       tty->print_cr("%4d of %4d @ " INTPTR_FORMAT ": Block len for %s",        \
  82                     len, alignment, e_addr-len, name);                         \
  83       guarantee(len <= alignment, "block too large");                          \
  84     }                                                                          \
  85     guarantee(len == e_addr-b_addr, "block len mismatch");                     \
  86   }
  87 #else
  88 // Macro to open a BranchTableBlock (a piece of code that is branched to by a calculated branch).
  89 #define BTB_BEGIN(lbl, alignment, name)                                        \
  90   __ align_address(alignment);                                                 \
  91   __ bind(lbl);                                                                \
  92   { unsigned int b_off = __ offset();                                          \
  93     uintptr_t   b_addr = (uintptr_t)__ pc();                                   \
  94     guarantee(b_addr%alignment == 0, "bad alignment at begin of block" name);
  95 
  96 // Macro to close a BranchTableBlock (a piece of code that is branched to by a calculated branch).
  97 #define BTB_END(lbl, alignment, name)                                          \
  98     uintptr_t   e_addr = (uintptr_t)__ pc();                                   \
  99     unsigned int e_off = __ offset();                                          \
 100     unsigned int len   = e_off-b_off;                                          \
 101     if (len > alignment) {                                                     \
 102       tty->print_cr("%4d of %4d @ " INTPTR_FORMAT ": Block len for %s",        \
 103                     len, alignment, e_addr-len, name);                         \
 104       guarantee(len <= alignment, "block too large");                          \
 105     }                                                                          \
 106     guarantee(len == e_addr-b_addr, "block len mismatch");                     \
 107   }
 108 #endif // ASSERT
 109 
 110 // Platform-dependent initialization.
 111 
 112 void TemplateTable::pd_initialize() {
 113   // No specific initialization.
 114 }
 115 
 116 // Address computation: local variables
 117 
 118 static inline Address iaddress(int n) {
 119   return Address(Z_locals, Interpreter::local_offset_in_bytes(n));
 120 }
 121 
 122 static inline Address laddress(int n) {
 123   return iaddress(n + 1);
 124 }
 125 
 126 static inline Address faddress(int n) {
 127   return iaddress(n);
 128 }
 129 
 130 static inline Address daddress(int n) {
 131   return laddress(n);
 132 }
 133 
 134 static inline Address aaddress(int n) {
 135   return iaddress(n);
 136 }
 137 
 138 // Pass NULL, if no shift instruction should be emitted.
 139 static inline Address iaddress(InterpreterMacroAssembler *masm, Register r) {
 140   if (masm) {
 141     masm->z_sllg(r, r, LogBytesPerWord);  // index2bytes
 142   }
 143   return Address(Z_locals, r, Interpreter::local_offset_in_bytes(0));
 144 }
 145 
 146 // Pass NULL, if no shift instruction should be emitted.
 147 static inline Address laddress(InterpreterMacroAssembler *masm, Register r) {
 148   if (masm) {
 149     masm->z_sllg(r, r, LogBytesPerWord);  // index2bytes
 150   }
 151   return Address(Z_locals, r, Interpreter::local_offset_in_bytes(1) );
 152 }
 153 
 154 static inline Address faddress(InterpreterMacroAssembler *masm, Register r) {
 155   return iaddress(masm, r);
 156 }
 157 
 158 static inline Address daddress(InterpreterMacroAssembler *masm, Register r) {
 159   return laddress(masm, r);
 160 }
 161 
 162 static inline Address aaddress(InterpreterMacroAssembler *masm, Register r) {
 163   return iaddress(masm, r);
 164 }
 165 
 166 // At top of Java expression stack which may be different than esp(). It
 167 // isn't for category 1 objects.
 168 static inline Address at_tos(int slot = 0) {
 169   return Address(Z_esp, Interpreter::expr_offset_in_bytes(slot));
 170 }
 171 
 172 // Condition conversion
 173 static Assembler::branch_condition j_not(TemplateTable::Condition cc) {
 174   switch (cc) {
 175     case TemplateTable::equal :
 176       return Assembler::bcondNotEqual;
 177     case TemplateTable::not_equal :
 178       return Assembler::bcondEqual;
 179     case TemplateTable::less :
 180       return Assembler::bcondNotLow;
 181     case TemplateTable::less_equal :
 182       return Assembler::bcondHigh;
 183     case TemplateTable::greater :
 184       return Assembler::bcondNotHigh;
 185     case TemplateTable::greater_equal:
 186       return Assembler::bcondLow;
 187   }
 188   ShouldNotReachHere();
 189   return Assembler::bcondZero;
 190 }
 191 
 192 // Do an oop store like *(base + offset) = val
 193 // offset can be a register or a constant.
 194 static void do_oop_store(InterpreterMacroAssembler* _masm,
 195                          Register base,
 196                          RegisterOrConstant offset,
 197                          Register val,
 198                          bool val_is_null, // == false does not guarantee that val really is not equal NULL.
 199                          Register tmp1,    // If tmp3 is volatile, either tmp1 or tmp2 must be
 200                          Register tmp2,    // non-volatile to hold a copy of pre_val across runtime calls.
 201                          Register tmp3,    // Ideally, this tmp register is non-volatile, as it is used to
 202                                            // hold pre_val (must survive runtime calls).
 203                          BarrierSet::Name barrier,
 204                          bool precise) {
 205   BLOCK_COMMENT("do_oop_store {");
 206   assert(val != noreg, "val must always be valid, even if it is zero");
 207   assert_different_registers(tmp1, tmp2, tmp3, val, base, offset.register_or_noreg());
 208   __ verify_oop(val);
 209   switch (barrier) {
 210 #if INCLUDE_ALL_GCS
 211     case BarrierSet::G1BarrierSet:
 212       {
 213 #ifdef ASSERT
 214         if (val_is_null) { // Check if the flag setting reflects reality.
 215           Label OK;
 216           __ z_ltgr(val, val);
 217           __ z_bre(OK);
 218           __ z_illtrap(0x11);
 219           __ bind(OK);
 220         }
 221 #endif
 222         Register pre_val = tmp3;
 223         // Load and record the previous value.
 224         __ g1_write_barrier_pre(base, offset, pre_val, val,
 225                                 tmp1, tmp2,
 226                                 false);  // Needs to hold pre_val in non_volatile register?
 227 
 228         if (val_is_null) {
 229           __ store_heap_oop_null(val, offset, base);
 230         } else {
 231           Label Done;
 232           // val_is_null == false does not guarantee that val really is not equal NULL.
 233           // Checking for this case dynamically has some cost, but also some benefit (in GC).
 234           // It's hard to say if cost or benefit is greater.
 235           { Label OK;
 236             __ z_ltgr(val, val);
 237             __ z_brne(OK);
 238             __ store_heap_oop_null(val, offset, base);
 239             __ z_bru(Done);
 240             __ bind(OK);
 241           }
 242           // G1 barrier needs uncompressed oop for region cross check.
 243           // Store_heap_oop compresses the oop in the argument register.
 244           Register val_work = val;
 245           if (UseCompressedOops) {
 246             val_work = tmp3;
 247             __ z_lgr(val_work, val);
 248           }
 249           __ store_heap_oop_not_null(val_work, offset, base);
 250 
 251           // We need precise card marks for oop array stores.
 252           // Otherwise, cardmarking the object which contains the oop is sufficient.
 253           if (precise && !(offset.is_constant() && offset.as_constant() == 0)) {
 254             __ add2reg_with_index(base,
 255                                   offset.constant_or_zero(),
 256                                   offset.register_or_noreg(),
 257                                   base);
 258           }
 259           __ g1_write_barrier_post(base /* store_adr */, val, tmp1, tmp2, tmp3);
 260           __ bind(Done);
 261         }
 262       }
 263       break;
 264 #endif // INCLUDE_ALL_GCS
 265     case BarrierSet::CardTableBarrierSet:
 266     {
 267       if (val_is_null) {
 268         __ store_heap_oop_null(val, offset, base);
 269       } else {
 270         __ store_heap_oop(val, offset, base);
 271         // Flatten object address if needed.
 272         if (precise && ((offset.register_or_noreg() != noreg) || (offset.constant_or_zero() != 0))) {
 273           __ load_address(base, Address(base, offset.register_or_noreg(), offset.constant_or_zero()));
 274         }
 275         __ card_write_barrier_post(base, tmp1);
 276       }
 277     }
 278     break;
 279   case BarrierSet::ModRef:
 280     // fall through
 281   default:
 282     ShouldNotReachHere();
 283 
 284   }
 285   BLOCK_COMMENT("} do_oop_store");
 286 }
 287 
 288 Address TemplateTable::at_bcp(int offset) {
 289   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 290   return Address(Z_bcp, offset);
 291 }
 292 
 293 void TemplateTable::patch_bytecode(Bytecodes::Code bc,
 294                                    Register        bc_reg,
 295                                    Register        temp_reg,
 296                                    bool            load_bc_into_bc_reg, // = true
 297                                    int             byte_no) {
 298   if (!RewriteBytecodes) { return; }
 299 
 300   NearLabel L_patch_done;
 301   BLOCK_COMMENT("patch_bytecode {");
 302 
 303   switch (bc) {
 304     case Bytecodes::_fast_aputfield:
 305     case Bytecodes::_fast_bputfield:
 306     case Bytecodes::_fast_zputfield:
 307     case Bytecodes::_fast_cputfield:
 308     case Bytecodes::_fast_dputfield:
 309     case Bytecodes::_fast_fputfield:
 310     case Bytecodes::_fast_iputfield:
 311     case Bytecodes::_fast_lputfield:
 312     case Bytecodes::_fast_sputfield:
 313       {
 314         // We skip bytecode quickening for putfield instructions when
 315         // the put_code written to the constant pool cache is zero.
 316         // This is required so that every execution of this instruction
 317         // calls out to InterpreterRuntime::resolve_get_put to do
 318         // additional, required work.
 319         assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 320         assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 321         __ get_cache_and_index_and_bytecode_at_bcp(Z_R1_scratch, bc_reg,
 322                                                    temp_reg, byte_no, 1);
 323         __ load_const_optimized(bc_reg, bc);
 324         __ compareU32_and_branch(temp_reg, (intptr_t)0,
 325                                  Assembler::bcondZero, L_patch_done);
 326       }
 327       break;
 328     default:
 329       assert(byte_no == -1, "sanity");
 330       // The pair bytecodes have already done the load.
 331       if (load_bc_into_bc_reg) {
 332         __ load_const_optimized(bc_reg, bc);
 333       }
 334       break;
 335   }
 336 
 337   if (JvmtiExport::can_post_breakpoint()) {
 338 
 339     Label   L_fast_patch;
 340 
 341     // If a breakpoint is present we can't rewrite the stream directly.
 342     __ z_cli(at_bcp(0), Bytecodes::_breakpoint);
 343     __ z_brne(L_fast_patch);
 344     __ get_method(temp_reg);
 345     // Let breakpoint table handling rewrite to quicker bytecode.
 346     __ call_VM_static(noreg,
 347                       CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at),
 348                       temp_reg, Z_R13, bc_reg);
 349     __ z_bru(L_patch_done);
 350 
 351     __ bind(L_fast_patch);
 352   }
 353 
 354 #ifdef ASSERT
 355   NearLabel   L_okay;
 356 
 357   // We load into 64 bits, since this works on any CPU.
 358   __ z_llgc(temp_reg, at_bcp(0));
 359   __ compareU32_and_branch(temp_reg, Bytecodes::java_code(bc),
 360                             Assembler::bcondEqual, L_okay        );
 361   __ compareU32_and_branch(temp_reg, bc_reg, Assembler::bcondEqual, L_okay);
 362   __ stop_static("patching the wrong bytecode");
 363   __ bind(L_okay);
 364 #endif
 365 
 366   // Patch bytecode.
 367   __ z_stc(bc_reg, at_bcp(0));
 368 
 369   __ bind(L_patch_done);
 370   BLOCK_COMMENT("} patch_bytecode");
 371 }
 372 
 373 // Individual instructions
 374 
 375 void TemplateTable::nop() {
 376   transition(vtos, vtos);
 377 }
 378 
 379 void TemplateTable::shouldnotreachhere() {
 380   transition(vtos, vtos);
 381   __ stop("shouldnotreachhere bytecode");
 382 }
 383 
 384 void TemplateTable::aconst_null() {
 385   transition(vtos, atos);
 386   __ clear_reg(Z_tos, true, false);
 387 }
 388 
 389 void TemplateTable::iconst(int value) {
 390   transition(vtos, itos);
 391   // Zero extension of the iconst makes zero extension at runtime obsolete.
 392   __ load_const_optimized(Z_tos, ((unsigned long)(unsigned int)value));
 393 }
 394 
 395 void TemplateTable::lconst(int value) {
 396   transition(vtos, ltos);
 397   __ load_const_optimized(Z_tos, value);
 398 }
 399 
 400 // No pc-relative load/store for floats.
 401 void TemplateTable::fconst(int value) {
 402   transition(vtos, ftos);
 403   static float   one = 1.0f, two = 2.0f;
 404 
 405   switch (value) {
 406     case 0:
 407       __ z_lzer(Z_ftos);
 408       return;
 409     case 1:
 410       __ load_absolute_address(Z_R1_scratch, (address) &one);
 411       __ mem2freg_opt(Z_ftos, Address(Z_R1_scratch), false);
 412       return;
 413     case 2:
 414       __ load_absolute_address(Z_R1_scratch, (address) &two);
 415       __ mem2freg_opt(Z_ftos, Address(Z_R1_scratch), false);
 416       return;
 417     default:
 418       ShouldNotReachHere();
 419       return;
 420   }
 421 }
 422 
 423 void TemplateTable::dconst(int value) {
 424   transition(vtos, dtos);
 425   static double one = 1.0;
 426 
 427   switch (value) {
 428     case 0:
 429       __ z_lzdr(Z_ftos);
 430       return;
 431     case 1:
 432       __ load_absolute_address(Z_R1_scratch, (address) &one);
 433       __ mem2freg_opt(Z_ftos, Address(Z_R1_scratch));
 434       return;
 435     default:
 436       ShouldNotReachHere();
 437       return;
 438   }
 439 }
 440 
 441 void TemplateTable::bipush() {
 442   transition(vtos, itos);
 443   __ z_lb(Z_tos, at_bcp(1));
 444 }
 445 
 446 void TemplateTable::sipush() {
 447   transition(vtos, itos);
 448   __ get_2_byte_integer_at_bcp(Z_tos, 1, InterpreterMacroAssembler::Signed);
 449 }
 450 
 451 
 452 void TemplateTable::ldc(bool wide) {
 453   transition(vtos, vtos);
 454   Label call_ldc, notFloat, notClass, notInt, Done;
 455   const Register RcpIndex = Z_tmp_1;
 456   const Register Rtags = Z_ARG2;
 457 
 458   if (wide) {
 459     __ get_2_byte_integer_at_bcp(RcpIndex, 1, InterpreterMacroAssembler::Unsigned);
 460   } else {
 461     __ z_llgc(RcpIndex, at_bcp(1));
 462   }
 463 
 464   __ get_cpool_and_tags(Z_tmp_2, Rtags);
 465 
 466   const int      base_offset = ConstantPool::header_size() * wordSize;
 467   const int      tags_offset = Array<u1>::base_offset_in_bytes();
 468   const Register Raddr_type = Rtags;
 469 
 470   // Get address of type.
 471   __ add2reg_with_index(Raddr_type, tags_offset, RcpIndex, Rtags);
 472 
 473   __ z_cli(0, Raddr_type, JVM_CONSTANT_UnresolvedClass);
 474   __ z_bre(call_ldc);    // Unresolved class - get the resolved class.
 475 
 476   __ z_cli(0, Raddr_type, JVM_CONSTANT_UnresolvedClassInError);
 477   __ z_bre(call_ldc);    // Unresolved class in error state - call into runtime
 478                          // to throw the error from the first resolution attempt.
 479 
 480   __ z_cli(0, Raddr_type, JVM_CONSTANT_Class);
 481   __ z_brne(notClass);   // Resolved class - need to call vm to get java
 482                          // mirror of the class.
 483 
 484   // We deal with a class. Call vm to do the appropriate.
 485   __ bind(call_ldc);
 486   __ load_const_optimized(Z_ARG2, wide);
 487   call_VM(Z_RET, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), Z_ARG2);
 488   __ push_ptr(Z_RET);
 489   __ z_bru(Done);
 490 
 491   // Not a class.
 492   __ bind(notClass);
 493   Register RcpOffset = RcpIndex;
 494   __ z_sllg(RcpOffset, RcpIndex, LogBytesPerWord); // Convert index to offset.
 495   __ z_cli(0, Raddr_type, JVM_CONSTANT_Float);
 496   __ z_brne(notFloat);
 497 
 498   // ftos
 499   __ mem2freg_opt(Z_ftos, Address(Z_tmp_2, RcpOffset, base_offset), false);
 500   __ push_f();
 501   __ z_bru(Done);
 502 
 503   __ bind(notFloat);
 504   __ z_cli(0, Raddr_type, JVM_CONSTANT_Integer);
 505   __ z_brne(notInt);
 506 
 507   // itos
 508   __ mem2reg_opt(Z_tos, Address(Z_tmp_2, RcpOffset, base_offset), false);
 509   __ push_i(Z_tos);
 510   __ z_bru(Done);
 511 
 512   // assume the tag is for condy; if not, the VM runtime will tell us
 513   __ bind(notInt);
 514   condy_helper(Done);
 515 
 516   __ bind(Done);
 517 }
 518 
 519 // Fast path for caching oop constants.
 520 // %%% We should use this to handle Class and String constants also.
 521 // %%% It will simplify the ldc/primitive path considerably.
 522 void TemplateTable::fast_aldc(bool wide) {
 523   transition(vtos, atos);
 524 
 525   const Register index = Z_tmp_2;
 526   int            index_size = wide ? sizeof(u2) : sizeof(u1);
 527   Label          L_do_resolve, L_resolved;
 528 
 529   // We are resolved if the resolved reference cache entry contains a
 530   // non-null object (CallSite, etc.).
 531   __ get_cache_index_at_bcp(index, 1, index_size);  // Load index.
 532   __ load_resolved_reference_at_index(Z_tos, index);
 533   __ z_ltgr(Z_tos, Z_tos);
 534   __ z_bre(L_do_resolve);
 535 
 536   // Convert null sentinel to NULL.
 537   __ load_const_optimized(Z_R1_scratch, (intptr_t)Universe::the_null_sentinel_addr());
 538   __ z_cg(Z_tos, Address(Z_R1_scratch));
 539   __ z_brne(L_resolved);
 540   __ clear_reg(Z_tos);
 541   __ z_bru(L_resolved);
 542 
 543   __ bind(L_do_resolve);
 544   // First time invocation - must resolve first.
 545   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 546   __ load_const_optimized(Z_ARG1, (int)bytecode());
 547   __ call_VM(Z_tos, entry, Z_ARG1);
 548 
 549   __ bind(L_resolved);
 550   __ verify_oop(Z_tos);
 551 }
 552 
 553 void TemplateTable::ldc2_w() {
 554   transition(vtos, vtos);
 555   Label notDouble, notLong, Done;
 556 
 557   // Z_tmp_1 = index of cp entry
 558   __ get_2_byte_integer_at_bcp(Z_tmp_1, 1, InterpreterMacroAssembler::Unsigned);
 559 
 560   __ get_cpool_and_tags(Z_tmp_2, Z_tos);
 561 
 562   const int base_offset = ConstantPool::header_size() * wordSize;
 563   const int tags_offset = Array<u1>::base_offset_in_bytes();
 564 
 565   // Get address of type.
 566   __ add2reg_with_index(Z_tos, tags_offset, Z_tos, Z_tmp_1);
 567 
 568   // Index needed in both branches, so calculate here.
 569   __ z_sllg(Z_tmp_1, Z_tmp_1, LogBytesPerWord);  // index2bytes
 570 
 571   // Check type.
 572   __ z_cli(0, Z_tos, JVM_CONSTANT_Double);
 573   __ z_brne(notDouble);
 574   // dtos
 575   __ mem2freg_opt(Z_ftos, Address(Z_tmp_2, Z_tmp_1, base_offset));
 576   __ push_d();
 577   __ z_bru(Done);
 578 
 579   __ bind(notDouble);
 580   __ z_cli(0, Z_tos, JVM_CONSTANT_Long);
 581   __ z_brne(notLong);
 582   // ltos
 583   __ mem2reg_opt(Z_tos, Address(Z_tmp_2, Z_tmp_1, base_offset));
 584   __ push_l();
 585   __ z_bru(Done);
 586 
 587   __ bind(notLong);
 588   condy_helper(Done);
 589 
 590   __ bind(Done);
 591 }
 592 
 593 void TemplateTable::condy_helper(Label& Done) {
 594   const Register obj   = Z_tmp_1;
 595   const Register off   = Z_tmp_2;
 596   const Register flags = Z_ARG1;
 597   const Register rarg  = Z_ARG2;
 598   __ load_const_optimized(rarg, (int)bytecode());
 599   call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc), rarg);
 600   __ get_vm_result_2(flags);
 601 
 602   // VMr = obj = base address to find primitive value to push
 603   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 604   assert(ConstantPoolCacheEntry::field_index_mask == 0xffff, "or use other instructions");
 605   __ z_llghr(off, flags);
 606   const Address field(obj, off);
 607 
 608   // What sort of thing are we loading?
 609   __ z_srl(flags, ConstantPoolCacheEntry::tos_state_shift);
 610   // Make sure we don't need to mask flags for tos_state after the above shift.
 611   ConstantPoolCacheEntry::verify_tos_state_shift();
 612 
 613   switch (bytecode()) {
 614   case Bytecodes::_ldc:
 615   case Bytecodes::_ldc_w:
 616     {
 617       // tos in (itos, ftos, stos, btos, ctos, ztos)
 618       Label notInt, notFloat, notShort, notByte, notChar, notBool;
 619       __ z_cghi(flags, itos);
 620       __ z_brne(notInt);
 621       // itos
 622       __ z_l(Z_tos, field);
 623       __ push(itos);
 624       __ z_bru(Done);
 625 
 626       __ bind(notInt);
 627       __ z_cghi(flags, ftos);
 628       __ z_brne(notFloat);
 629       // ftos
 630       __ z_le(Z_ftos, field);
 631       __ push(ftos);
 632       __ z_bru(Done);
 633 
 634       __ bind(notFloat);
 635       __ z_cghi(flags, stos);
 636       __ z_brne(notShort);
 637       // stos
 638       __ z_lh(Z_tos, field);
 639       __ push(stos);
 640       __ z_bru(Done);
 641 
 642       __ bind(notShort);
 643       __ z_cghi(flags, btos);
 644       __ z_brne(notByte);
 645       // btos
 646       __ z_lb(Z_tos, field);
 647       __ push(btos);
 648       __ z_bru(Done);
 649 
 650       __ bind(notByte);
 651       __ z_cghi(flags, ctos);
 652       __ z_brne(notChar);
 653       // ctos
 654       __ z_llh(Z_tos, field);
 655       __ push(ctos);
 656       __ z_bru(Done);
 657 
 658       __ bind(notChar);
 659       __ z_cghi(flags, ztos);
 660       __ z_brne(notBool);
 661       // ztos
 662       __ z_lb(Z_tos, field);
 663       __ push(ztos);
 664       __ z_bru(Done);
 665 
 666       __ bind(notBool);
 667       break;
 668     }
 669 
 670   case Bytecodes::_ldc2_w:
 671     {
 672       Label notLong, notDouble;
 673       __ z_cghi(flags, ltos);
 674       __ z_brne(notLong);
 675       // ltos
 676       __ z_lg(Z_tos, field);
 677       __ push(ltos);
 678       __ z_bru(Done);
 679 
 680       __ bind(notLong);
 681       __ z_cghi(flags, dtos);
 682       __ z_brne(notDouble);
 683       // dtos
 684       __ z_ld(Z_ftos, field);
 685       __ push(dtos);
 686       __ z_bru(Done);
 687 
 688       __ bind(notDouble);
 689       break;
 690     }
 691 
 692   default:
 693     ShouldNotReachHere();
 694   }
 695 
 696   __ stop("bad ldc/condy");
 697 }
 698 
 699 void TemplateTable::locals_index(Register reg, int offset) {
 700   __ z_llgc(reg, at_bcp(offset));
 701   __ z_lcgr(reg);
 702 }
 703 
 704 void TemplateTable::iload() {
 705   iload_internal();
 706 }
 707 
 708 void TemplateTable::nofast_iload() {
 709   iload_internal(may_not_rewrite);
 710 }
 711 
 712 void TemplateTable::iload_internal(RewriteControl rc) {
 713   transition(vtos, itos);
 714 
 715   if (RewriteFrequentPairs && rc == may_rewrite) {
 716     NearLabel rewrite, done;
 717     const Register bc = Z_ARG4;
 718 
 719     assert(Z_R1_scratch != bc, "register damaged");
 720 
 721     // Get next byte.
 722     __ z_llgc(Z_R1_scratch, at_bcp(Bytecodes::length_for (Bytecodes::_iload)));
 723 
 724     // If _iload, wait to rewrite to iload2. We only want to rewrite the
 725     // last two iloads in a pair. Comparing against fast_iload means that
 726     // the next bytecode is neither an iload or a caload, and therefore
 727     // an iload pair.
 728     __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_iload,
 729                              Assembler::bcondEqual, done);
 730 
 731     __ load_const_optimized(bc, Bytecodes::_fast_iload2);
 732     __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_fast_iload,
 733                              Assembler::bcondEqual, rewrite);
 734 
 735     // If _caload, rewrite to fast_icaload.
 736     __ load_const_optimized(bc, Bytecodes::_fast_icaload);
 737     __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_caload,
 738                              Assembler::bcondEqual, rewrite);
 739 
 740     // Rewrite so iload doesn't check again.
 741     __ load_const_optimized(bc, Bytecodes::_fast_iload);
 742 
 743     // rewrite
 744     // bc: fast bytecode
 745     __ bind(rewrite);
 746     patch_bytecode(Bytecodes::_iload, bc, Z_R1_scratch, false);
 747 
 748     __ bind(done);
 749 
 750   }
 751 
 752   // Get the local value into tos.
 753   locals_index(Z_R1_scratch);
 754   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
 755 }
 756 
 757 void TemplateTable::fast_iload2() {
 758   transition(vtos, itos);
 759 
 760   locals_index(Z_R1_scratch);
 761   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
 762   __ push_i(Z_tos);
 763   locals_index(Z_R1_scratch, 3);
 764   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
 765 }
 766 
 767 void TemplateTable::fast_iload() {
 768   transition(vtos, itos);
 769 
 770   locals_index(Z_R1_scratch);
 771   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
 772 }
 773 
 774 void TemplateTable::lload() {
 775   transition(vtos, ltos);
 776 
 777   locals_index(Z_R1_scratch);
 778   __ mem2reg_opt(Z_tos, laddress(_masm, Z_R1_scratch));
 779 }
 780 
 781 void TemplateTable::fload() {
 782   transition(vtos, ftos);
 783 
 784   locals_index(Z_R1_scratch);
 785   __ mem2freg_opt(Z_ftos, faddress(_masm, Z_R1_scratch), false);
 786 }
 787 
 788 void TemplateTable::dload() {
 789   transition(vtos, dtos);
 790 
 791   locals_index(Z_R1_scratch);
 792   __ mem2freg_opt(Z_ftos, daddress(_masm, Z_R1_scratch));
 793 }
 794 
 795 void TemplateTable::aload() {
 796   transition(vtos, atos);
 797 
 798   locals_index(Z_R1_scratch);
 799   __ mem2reg_opt(Z_tos, aaddress(_masm, Z_R1_scratch));
 800 }
 801 
 802 void TemplateTable::locals_index_wide(Register reg) {
 803   __ get_2_byte_integer_at_bcp(reg, 2, InterpreterMacroAssembler::Unsigned);
 804   __ z_lcgr(reg);
 805 }
 806 
 807 void TemplateTable::wide_iload() {
 808   transition(vtos, itos);
 809 
 810   locals_index_wide(Z_tmp_1);
 811   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_tmp_1), false);
 812 }
 813 
 814 void TemplateTable::wide_lload() {
 815   transition(vtos, ltos);
 816 
 817   locals_index_wide(Z_tmp_1);
 818   __ mem2reg_opt(Z_tos, laddress(_masm, Z_tmp_1));
 819 }
 820 
 821 void TemplateTable::wide_fload() {
 822   transition(vtos, ftos);
 823 
 824   locals_index_wide(Z_tmp_1);
 825   __ mem2freg_opt(Z_ftos, faddress(_masm, Z_tmp_1), false);
 826 }
 827 
 828 void TemplateTable::wide_dload() {
 829   transition(vtos, dtos);
 830 
 831   locals_index_wide(Z_tmp_1);
 832   __ mem2freg_opt(Z_ftos, daddress(_masm, Z_tmp_1));
 833 }
 834 
 835 void TemplateTable::wide_aload() {
 836   transition(vtos, atos);
 837 
 838   locals_index_wide(Z_tmp_1);
 839   __ mem2reg_opt(Z_tos, aaddress(_masm, Z_tmp_1));
 840 }
 841 
 842 void TemplateTable::index_check(Register array, Register index, unsigned int shift) {
 843   assert_different_registers(Z_R1_scratch, array, index);
 844 
 845   // Check array.
 846   __ null_check(array, Z_R0_scratch, arrayOopDesc::length_offset_in_bytes());
 847 
 848   // Sign extend index for use by indexed load.
 849   __ z_lgfr(index, index);
 850 
 851   // Check index.
 852   Label index_ok;
 853   __ z_cl(index, Address(array, arrayOopDesc::length_offset_in_bytes()));
 854   __ z_brl(index_ok);
 855   __ lgr_if_needed(Z_ARG3, index); // See generate_ArrayIndexOutOfBounds_handler().
 856   // Give back the array to create more detailed exceptions.
 857   __ lgr_if_needed(Z_ARG2, array); // See generate_ArrayIndexOutOfBounds_handler().
 858   __ load_absolute_address(Z_R1_scratch,
 859                            Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 860   __ z_bcr(Assembler::bcondAlways, Z_R1_scratch);
 861   __ bind(index_ok);
 862 
 863   if (shift > 0)
 864     __ z_sllg(index, index, shift);
 865 }
 866 
 867 void TemplateTable::iaload() {
 868   transition(itos, itos);
 869 
 870   __ pop_ptr(Z_tmp_1);  // array
 871   // Index is in Z_tos.
 872   Register index = Z_tos;
 873   index_check(Z_tmp_1, index, LogBytesPerInt); // Kills Z_ARG3.
 874   // Load the value.
 875   __ mem2reg_opt(Z_tos,
 876                  Address(Z_tmp_1, index, arrayOopDesc::base_offset_in_bytes(T_INT)),
 877                  false);
 878 }
 879 
 880 void TemplateTable::laload() {
 881   transition(itos, ltos);
 882 
 883   __ pop_ptr(Z_tmp_2);
 884   // Z_tos   : index
 885   // Z_tmp_2 : array
 886   Register index = Z_tos;
 887   index_check(Z_tmp_2, index, LogBytesPerLong);
 888   __ mem2reg_opt(Z_tos,
 889                  Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_LONG)));
 890 }
 891 
 892 void TemplateTable::faload() {
 893   transition(itos, ftos);
 894 
 895   __ pop_ptr(Z_tmp_2);
 896   // Z_tos   : index
 897   // Z_tmp_2 : array
 898   Register index = Z_tos;
 899   index_check(Z_tmp_2, index, LogBytesPerInt);
 900   __ mem2freg_opt(Z_ftos,
 901                   Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_FLOAT)),
 902                   false);
 903 }
 904 
 905 void TemplateTable::daload() {
 906   transition(itos, dtos);
 907 
 908   __ pop_ptr(Z_tmp_2);
 909   // Z_tos   : index
 910   // Z_tmp_2 : array
 911   Register index = Z_tos;
 912   index_check(Z_tmp_2, index, LogBytesPerLong);
 913   __ mem2freg_opt(Z_ftos,
 914                   Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
 915 }
 916 
 917 void TemplateTable::aaload() {
 918   transition(itos, atos);
 919 
 920   unsigned const int shift = LogBytesPerHeapOop;
 921   __ pop_ptr(Z_tmp_1);  // array
 922   // Index is in Z_tos.
 923   Register index = Z_tos;
 924   index_check(Z_tmp_1, index, shift);
 925   // Now load array element.
 926   __ load_heap_oop(Z_tos,
 927                    Address(Z_tmp_1, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 928   __ verify_oop(Z_tos);
 929 }
 930 
 931 void TemplateTable::baload() {
 932   transition(itos, itos);
 933 
 934   __ pop_ptr(Z_tmp_1);
 935   // Z_tos   : index
 936   // Z_tmp_1 : array
 937   Register index = Z_tos;
 938   index_check(Z_tmp_1, index, 0);
 939   __ z_lb(Z_tos,
 940           Address(Z_tmp_1, index, arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 941 }
 942 
 943 void TemplateTable::caload() {
 944   transition(itos, itos);
 945 
 946   __ pop_ptr(Z_tmp_2);
 947   // Z_tos   : index
 948   // Z_tmp_2 : array
 949   Register index = Z_tos;
 950   index_check(Z_tmp_2, index, LogBytesPerShort);
 951   // Load into 64 bits, works on all CPUs.
 952   __ z_llgh(Z_tos,
 953             Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 954 }
 955 
 956 // Iload followed by caload frequent pair.
 957 void TemplateTable::fast_icaload() {
 958   transition(vtos, itos);
 959 
 960   // Load index out of locals.
 961   locals_index(Z_R1_scratch);
 962   __ mem2reg_opt(Z_ARG3, iaddress(_masm, Z_R1_scratch), false);
 963   // Z_ARG3  : index
 964   // Z_tmp_2 : array
 965   __ pop_ptr(Z_tmp_2);
 966   index_check(Z_tmp_2, Z_ARG3, LogBytesPerShort);
 967   // Load into 64 bits, works on all CPUs.
 968   __ z_llgh(Z_tos,
 969             Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 970 }
 971 
 972 void TemplateTable::saload() {
 973   transition(itos, itos);
 974 
 975   __ pop_ptr(Z_tmp_2);
 976   // Z_tos   : index
 977   // Z_tmp_2 : array
 978   Register index = Z_tos;
 979   index_check(Z_tmp_2, index, LogBytesPerShort);
 980   __ z_lh(Z_tos,
 981           Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 982 }
 983 
 984 void TemplateTable::iload(int n) {
 985   transition(vtos, itos);
 986   __ z_ly(Z_tos, iaddress(n));
 987 }
 988 
 989 void TemplateTable::lload(int n) {
 990   transition(vtos, ltos);
 991   __ z_lg(Z_tos, laddress(n));
 992 }
 993 
 994 void TemplateTable::fload(int n) {
 995   transition(vtos, ftos);
 996   __ mem2freg_opt(Z_ftos, faddress(n), false);
 997 }
 998 
 999 void TemplateTable::dload(int n) {
1000   transition(vtos, dtos);
1001   __ mem2freg_opt(Z_ftos, daddress(n));
1002 }
1003 
1004 void TemplateTable::aload(int n) {
1005   transition(vtos, atos);
1006   __ mem2reg_opt(Z_tos, aaddress(n));
1007 }
1008 
1009 void TemplateTable::aload_0() {
1010   aload_0_internal();
1011 }
1012 
1013 void TemplateTable::nofast_aload_0() {
1014   aload_0_internal(may_not_rewrite);
1015 }
1016 
1017 void TemplateTable::aload_0_internal(RewriteControl rc) {
1018   transition(vtos, atos);
1019 
1020   // According to bytecode histograms, the pairs:
1021   //
1022   // _aload_0, _fast_igetfield
1023   // _aload_0, _fast_agetfield
1024   // _aload_0, _fast_fgetfield
1025   //
1026   // occur frequently. If RewriteFrequentPairs is set, the (slow)
1027   // _aload_0 bytecode checks if the next bytecode is either
1028   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
1029   // rewrites the current bytecode into a pair bytecode; otherwise it
1030   // rewrites the current bytecode into _fast_aload_0 that doesn't do
1031   // the pair check anymore.
1032   //
1033   // Note: If the next bytecode is _getfield, the rewrite must be
1034   //       delayed, otherwise we may miss an opportunity for a pair.
1035   //
1036   // Also rewrite frequent pairs
1037   //   aload_0, aload_1
1038   //   aload_0, iload_1
1039   // These bytecodes with a small amount of code are most profitable
1040   // to rewrite.
1041   if (!(RewriteFrequentPairs && (rc == may_rewrite))) {
1042     aload(0);
1043     return;
1044   }
1045 
1046   NearLabel rewrite, done;
1047   const Register bc = Z_ARG4;
1048 
1049   assert(Z_R1_scratch != bc, "register damaged");
1050   // Get next byte.
1051   __ z_llgc(Z_R1_scratch, at_bcp(Bytecodes::length_for (Bytecodes::_aload_0)));
1052 
1053   // Do actual aload_0.
1054   aload(0);
1055 
1056   // If _getfield then wait with rewrite.
1057   __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_getfield,
1058                            Assembler::bcondEqual, done);
1059 
1060   // If _igetfield then rewrite to _fast_iaccess_0.
1061   assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0)
1062             == Bytecodes::_aload_0, "fix bytecode definition");
1063 
1064   __ load_const_optimized(bc, Bytecodes::_fast_iaccess_0);
1065   __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_fast_igetfield,
1066                            Assembler::bcondEqual, rewrite);
1067 
1068   // If _agetfield then rewrite to _fast_aaccess_0.
1069   assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0)
1070             == Bytecodes::_aload_0, "fix bytecode definition");
1071 
1072   __ load_const_optimized(bc, Bytecodes::_fast_aaccess_0);
1073   __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_fast_agetfield,
1074                            Assembler::bcondEqual, rewrite);
1075 
1076   // If _fgetfield then rewrite to _fast_faccess_0.
1077   assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0)
1078             == Bytecodes::_aload_0, "fix bytecode definition");
1079 
1080   __ load_const_optimized(bc, Bytecodes::_fast_faccess_0);
1081   __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_fast_fgetfield,
1082                            Assembler::bcondEqual, rewrite);
1083 
1084   // Else rewrite to _fast_aload0.
1085   assert(Bytecodes::java_code(Bytecodes::_fast_aload_0)
1086             == Bytecodes::_aload_0, "fix bytecode definition");
1087   __ load_const_optimized(bc, Bytecodes::_fast_aload_0);
1088 
1089   // rewrite
1090   // bc: fast bytecode
1091   __ bind(rewrite);
1092 
1093   patch_bytecode(Bytecodes::_aload_0, bc, Z_R1_scratch, false);
1094   // Reload local 0 because of VM call inside patch_bytecode().
1095   // this may trigger GC and thus change the oop.
1096   aload(0);
1097 
1098   __ bind(done);
1099 }
1100 
1101 void TemplateTable::istore() {
1102   transition(itos, vtos);
1103   locals_index(Z_R1_scratch);
1104   __ reg2mem_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
1105 }
1106 
1107 void TemplateTable::lstore() {
1108   transition(ltos, vtos);
1109   locals_index(Z_R1_scratch);
1110   __ reg2mem_opt(Z_tos, laddress(_masm, Z_R1_scratch));
1111 }
1112 
1113 void TemplateTable::fstore() {
1114   transition(ftos, vtos);
1115   locals_index(Z_R1_scratch);
1116   __ freg2mem_opt(Z_ftos, faddress(_masm, Z_R1_scratch));
1117 }
1118 
1119 void TemplateTable::dstore() {
1120   transition(dtos, vtos);
1121   locals_index(Z_R1_scratch);
1122   __ freg2mem_opt(Z_ftos, daddress(_masm, Z_R1_scratch));
1123 }
1124 
1125 void TemplateTable::astore() {
1126   transition(vtos, vtos);
1127   __ pop_ptr(Z_tos);
1128   locals_index(Z_R1_scratch);
1129   __ reg2mem_opt(Z_tos, aaddress(_masm, Z_R1_scratch));
1130 }
1131 
1132 void TemplateTable::wide_istore() {
1133   transition(vtos, vtos);
1134   __ pop_i(Z_tos);
1135   locals_index_wide(Z_tmp_1);
1136   __ reg2mem_opt(Z_tos, iaddress(_masm, Z_tmp_1), false);
1137 }
1138 
1139 void TemplateTable::wide_lstore() {
1140   transition(vtos, vtos);
1141   __ pop_l(Z_tos);
1142   locals_index_wide(Z_tmp_1);
1143   __ reg2mem_opt(Z_tos, laddress(_masm, Z_tmp_1));
1144 }
1145 
1146 void TemplateTable::wide_fstore() {
1147   transition(vtos, vtos);
1148   __ pop_f(Z_ftos);
1149   locals_index_wide(Z_tmp_1);
1150   __ freg2mem_opt(Z_ftos, faddress(_masm, Z_tmp_1), false);
1151 }
1152 
1153 void TemplateTable::wide_dstore() {
1154   transition(vtos, vtos);
1155   __ pop_d(Z_ftos);
1156   locals_index_wide(Z_tmp_1);
1157   __ freg2mem_opt(Z_ftos, daddress(_masm, Z_tmp_1));
1158 }
1159 
1160 void TemplateTable::wide_astore() {
1161   transition(vtos, vtos);
1162   __ pop_ptr(Z_tos);
1163   locals_index_wide(Z_tmp_1);
1164   __ reg2mem_opt(Z_tos, aaddress(_masm, Z_tmp_1));
1165 }
1166 
1167 void TemplateTable::iastore() {
1168   transition(itos, vtos);
1169 
1170   Register index = Z_ARG3; // Index_check expects index in Z_ARG3.
1171   // Value is in Z_tos ...
1172   __ pop_i(index);        // index
1173   __ pop_ptr(Z_tmp_1);    // array
1174   index_check(Z_tmp_1, index, LogBytesPerInt);
1175   // ... and then move the value.
1176   __ reg2mem_opt(Z_tos,
1177                  Address(Z_tmp_1, index, arrayOopDesc::base_offset_in_bytes(T_INT)),
1178                  false);
1179 }
1180 
1181 void TemplateTable::lastore() {
1182   transition(ltos, vtos);
1183 
1184   __ pop_i(Z_ARG3);
1185   __ pop_ptr(Z_tmp_2);
1186   // Z_tos   : value
1187   // Z_ARG3  : index
1188   // Z_tmp_2 : array
1189  index_check(Z_tmp_2, Z_ARG3, LogBytesPerLong); // Prefer index in Z_ARG3.
1190   __ reg2mem_opt(Z_tos,
1191                  Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_LONG)));
1192 }
1193 
1194 void TemplateTable::fastore() {
1195   transition(ftos, vtos);
1196 
1197   __ pop_i(Z_ARG3);
1198   __ pop_ptr(Z_tmp_2);
1199   // Z_ftos  : value
1200   // Z_ARG3  : index
1201   // Z_tmp_2 : array
1202   index_check(Z_tmp_2, Z_ARG3, LogBytesPerInt); // Prefer index in Z_ARG3.
1203   __ freg2mem_opt(Z_ftos,
1204                   Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_FLOAT)),
1205                   false);
1206 }
1207 
1208 void TemplateTable::dastore() {
1209   transition(dtos, vtos);
1210 
1211   __ pop_i(Z_ARG3);
1212   __ pop_ptr(Z_tmp_2);
1213   // Z_ftos  : value
1214   // Z_ARG3  : index
1215   // Z_tmp_2 : array
1216   index_check(Z_tmp_2, Z_ARG3, LogBytesPerLong); // Prefer index in Z_ARG3.
1217   __ freg2mem_opt(Z_ftos,
1218                   Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1219 }
1220 
1221 void TemplateTable::aastore() {
1222   NearLabel is_null, ok_is_subtype, done;
1223   transition(vtos, vtos);
1224 
1225   // stack: ..., array, index, value
1226 
1227   Register Rvalue = Z_tos;
1228   Register Rarray = Z_ARG2;
1229   Register Rindex = Z_ARG3; // Convention for index_check().
1230 
1231   __ load_ptr(0, Rvalue);
1232   __ z_l(Rindex, Address(Z_esp, Interpreter::expr_offset_in_bytes(1)));
1233   __ load_ptr(2, Rarray);
1234 
1235   unsigned const int shift = LogBytesPerHeapOop;
1236   index_check(Rarray, Rindex, shift); // side effect: Rindex = Rindex << shift
1237   Register Rstore_addr  = Rindex;
1238   // Address where the store goes to, i.e. &(Rarry[index])
1239   __ load_address(Rstore_addr, Address(Rarray, Rindex, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1240 
1241   // do array store check - check for NULL value first.
1242   __ compareU64_and_branch(Rvalue, (intptr_t)0, Assembler::bcondEqual, is_null);
1243 
1244   Register Rsub_klass   = Z_ARG4;
1245   Register Rsuper_klass = Z_ARG5;
1246   __ load_klass(Rsub_klass, Rvalue);
1247   // Load superklass.
1248   __ load_klass(Rsuper_klass, Rarray);
1249   __ z_lg(Rsuper_klass, Address(Rsuper_klass, ObjArrayKlass::element_klass_offset()));
1250 
1251   // Generate a fast subtype check.  Branch to ok_is_subtype if no failure.
1252   // Throw if failure.
1253   Register tmp1 = Z_tmp_1;
1254   Register tmp2 = Z_tmp_2;
1255   __ gen_subtype_check(Rsub_klass, Rsuper_klass, tmp1, tmp2, ok_is_subtype);
1256 
1257   // Fall through on failure.
1258   // Object is in Rvalue == Z_tos.
1259   assert(Rvalue == Z_tos, "that's the expected location");
1260   __ load_absolute_address(tmp1, Interpreter::_throw_ArrayStoreException_entry);
1261   __ z_br(tmp1);
1262 
1263   // Come here on success.
1264   __ bind(ok_is_subtype);
1265 
1266   // Now store using the appropriate barrier.
1267   Register tmp3 = Rsub_klass;
1268   do_oop_store(_masm, Rstore_addr, (intptr_t)0/*offset*/, Rvalue, false/*val==null*/,
1269                tmp3, tmp2, tmp1, _bs->kind(), true);
1270   __ z_bru(done);
1271 
1272   // Have a NULL in Rvalue.
1273   __ bind(is_null);
1274   __ profile_null_seen(tmp1);
1275 
1276   // Store a NULL.
1277   do_oop_store(_masm, Rstore_addr, (intptr_t)0/*offset*/, Rvalue, true/*val==null*/,
1278                tmp3, tmp2, tmp1, _bs->kind(), true);
1279 
1280   // Pop stack arguments.
1281   __ bind(done);
1282   __ add2reg(Z_esp, 3 * Interpreter::stackElementSize);
1283 }
1284 
1285 
1286 void TemplateTable::bastore() {
1287   transition(itos, vtos);
1288 
1289   __ pop_i(Z_ARG3);
1290   __ pop_ptr(Z_tmp_2);
1291   // Z_tos   : value
1292   // Z_ARG3  : index
1293   // Z_tmp_2 : array
1294 
1295   // Need to check whether array is boolean or byte
1296   // since both types share the bastore bytecode.
1297   __ load_klass(Z_tmp_1, Z_tmp_2);
1298   __ z_llgf(Z_tmp_1, Address(Z_tmp_1, Klass::layout_helper_offset()));
1299   __ z_tmll(Z_tmp_1, Klass::layout_helper_boolean_diffbit());
1300   Label L_skip;
1301   __ z_bfalse(L_skip);
1302   // if it is a T_BOOLEAN array, mask the stored value to 0/1
1303   __ z_nilf(Z_tos, 0x1);
1304   __ bind(L_skip);
1305 
1306   // No index shift necessary - pass 0.
1307   index_check(Z_tmp_2, Z_ARG3, 0); // Prefer index in Z_ARG3.
1308   __ z_stc(Z_tos,
1309            Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1310 }
1311 
1312 void TemplateTable::castore() {
1313   transition(itos, vtos);
1314 
1315   __ pop_i(Z_ARG3);
1316   __ pop_ptr(Z_tmp_2);
1317   // Z_tos   : value
1318   // Z_ARG3  : index
1319   // Z_tmp_2 : array
1320   Register index = Z_ARG3; // prefer index in Z_ARG3
1321   index_check(Z_tmp_2, index, LogBytesPerShort);
1322   __ z_sth(Z_tos,
1323            Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1324 }
1325 
1326 void TemplateTable::sastore() {
1327   castore();
1328 }
1329 
1330 void TemplateTable::istore(int n) {
1331   transition(itos, vtos);
1332   __ reg2mem_opt(Z_tos, iaddress(n), false);
1333 }
1334 
1335 void TemplateTable::lstore(int n) {
1336   transition(ltos, vtos);
1337   __ reg2mem_opt(Z_tos, laddress(n));
1338 }
1339 
1340 void TemplateTable::fstore(int n) {
1341   transition(ftos, vtos);
1342   __ freg2mem_opt(Z_ftos, faddress(n), false);
1343 }
1344 
1345 void TemplateTable::dstore(int n) {
1346   transition(dtos, vtos);
1347   __ freg2mem_opt(Z_ftos, daddress(n));
1348 }
1349 
1350 void TemplateTable::astore(int n) {
1351   transition(vtos, vtos);
1352   __ pop_ptr(Z_tos);
1353   __ reg2mem_opt(Z_tos, aaddress(n));
1354 }
1355 
1356 void TemplateTable::pop() {
1357   transition(vtos, vtos);
1358   __ add2reg(Z_esp, Interpreter::stackElementSize);
1359 }
1360 
1361 void TemplateTable::pop2() {
1362   transition(vtos, vtos);
1363   __ add2reg(Z_esp, 2 * Interpreter::stackElementSize);
1364 }
1365 
1366 void TemplateTable::dup() {
1367   transition(vtos, vtos);
1368   __ load_ptr(0, Z_tos);
1369   __ push_ptr(Z_tos);
1370   // stack: ..., a, a
1371 }
1372 
1373 void TemplateTable::dup_x1() {
1374   transition(vtos, vtos);
1375 
1376   // stack: ..., a, b
1377   __ load_ptr(0, Z_tos);          // load b
1378   __ load_ptr(1, Z_R0_scratch);   // load a
1379   __ store_ptr(1, Z_tos);         // store b
1380   __ store_ptr(0, Z_R0_scratch);  // store a
1381   __ push_ptr(Z_tos);             // push b
1382   // stack: ..., b, a, b
1383 }
1384 
1385 void TemplateTable::dup_x2() {
1386   transition(vtos, vtos);
1387 
1388   // stack: ..., a, b, c
1389   __ load_ptr(0, Z_R0_scratch);   // load c
1390   __ load_ptr(2, Z_R1_scratch);   // load a
1391   __ store_ptr(2, Z_R0_scratch);  // store c in a
1392   __ push_ptr(Z_R0_scratch);      // push c
1393   // stack: ..., c, b, c, c
1394   __ load_ptr(2, Z_R0_scratch);   // load b
1395   __ store_ptr(2, Z_R1_scratch);  // store a in b
1396   // stack: ..., c, a, c, c
1397   __ store_ptr(1, Z_R0_scratch);  // store b in c
1398   // stack: ..., c, a, b, c
1399 }
1400 
1401 void TemplateTable::dup2() {
1402   transition(vtos, vtos);
1403 
1404   // stack: ..., a, b
1405   __ load_ptr(1, Z_R0_scratch);  // load a
1406   __ push_ptr(Z_R0_scratch);     // push a
1407   __ load_ptr(1, Z_R0_scratch);  // load b
1408   __ push_ptr(Z_R0_scratch);     // push b
1409   // stack: ..., a, b, a, b
1410 }
1411 
1412 void TemplateTable::dup2_x1() {
1413   transition(vtos, vtos);
1414 
1415   // stack: ..., a, b, c
1416   __ load_ptr(0, Z_R0_scratch);  // load c
1417   __ load_ptr(1, Z_R1_scratch);  // load b
1418   __ push_ptr(Z_R1_scratch);     // push b
1419   __ push_ptr(Z_R0_scratch);     // push c
1420   // stack: ..., a, b, c, b, c
1421   __ store_ptr(3, Z_R0_scratch); // store c in b
1422   // stack: ..., a, c, c, b, c
1423   __ load_ptr( 4, Z_R0_scratch); // load a
1424   __ store_ptr(2, Z_R0_scratch); // store a in 2nd c
1425   // stack: ..., a, c, a, b, c
1426   __ store_ptr(4, Z_R1_scratch); // store b in a
1427   // stack: ..., b, c, a, b, c
1428 }
1429 
1430 void TemplateTable::dup2_x2() {
1431   transition(vtos, vtos);
1432 
1433   // stack: ..., a, b, c, d
1434   __ load_ptr(0, Z_R0_scratch);   // load d
1435   __ load_ptr(1, Z_R1_scratch);   // load c
1436   __ push_ptr(Z_R1_scratch);      // push c
1437   __ push_ptr(Z_R0_scratch);      // push d
1438   // stack: ..., a, b, c, d, c, d
1439   __ load_ptr(4, Z_R1_scratch);   // load b
1440   __ store_ptr(2, Z_R1_scratch);  // store b in d
1441   __ store_ptr(4, Z_R0_scratch);  // store d in b
1442   // stack: ..., a, d, c, b, c, d
1443   __ load_ptr(5, Z_R0_scratch);   // load a
1444   __ load_ptr(3, Z_R1_scratch);   // load c
1445   __ store_ptr(3, Z_R0_scratch);  // store a in c
1446   __ store_ptr(5, Z_R1_scratch);  // store c in a
1447   // stack: ..., c, d, a, b, c, d
1448 }
1449 
1450 void TemplateTable::swap() {
1451   transition(vtos, vtos);
1452 
1453   // stack: ..., a, b
1454   __ load_ptr(1, Z_R0_scratch);  // load a
1455   __ load_ptr(0, Z_R1_scratch);  // load b
1456   __ store_ptr(0, Z_R0_scratch);  // store a in b
1457   __ store_ptr(1, Z_R1_scratch);  // store b in a
1458   // stack: ..., b, a
1459 }
1460 
1461 void TemplateTable::iop2(Operation op) {
1462   transition(itos, itos);
1463   switch (op) {
1464     case add  :                           __ z_ay(Z_tos,  __ stackTop()); __ pop_i(); break;
1465     case sub  :                           __ z_sy(Z_tos,  __ stackTop()); __ pop_i(); __ z_lcr(Z_tos, Z_tos); break;
1466     case mul  :                           __ z_msy(Z_tos, __ stackTop()); __ pop_i(); break;
1467     case _and :                           __ z_ny(Z_tos,  __ stackTop()); __ pop_i(); break;
1468     case _or  :                           __ z_oy(Z_tos,  __ stackTop()); __ pop_i(); break;
1469     case _xor :                           __ z_xy(Z_tos,  __ stackTop()); __ pop_i(); break;
1470     case shl  : __ z_lr(Z_tmp_1, Z_tos);
1471                 __ z_nill(Z_tmp_1, 31);  // Lowest 5 bits are shiftamount.
1472                                           __ pop_i(Z_tos);   __ z_sll(Z_tos, 0,  Z_tmp_1); break;
1473     case shr  : __ z_lr(Z_tmp_1, Z_tos);
1474                 __ z_nill(Z_tmp_1, 31);  // Lowest 5 bits are shiftamount.
1475                                           __ pop_i(Z_tos);   __ z_sra(Z_tos, 0,  Z_tmp_1); break;
1476     case ushr : __ z_lr(Z_tmp_1, Z_tos);
1477                 __ z_nill(Z_tmp_1, 31);  // Lowest 5 bits are shiftamount.
1478                                           __ pop_i(Z_tos);   __ z_srl(Z_tos, 0,  Z_tmp_1); break;
1479     default   : ShouldNotReachHere(); break;
1480   }
1481   return;
1482 }
1483 
1484 void TemplateTable::lop2(Operation op) {
1485   transition(ltos, ltos);
1486 
1487   switch (op) {
1488     case add  :  __ z_ag(Z_tos,  __ stackTop()); __ pop_l(); break;
1489     case sub  :  __ z_sg(Z_tos,  __ stackTop()); __ pop_l(); __ z_lcgr(Z_tos, Z_tos); break;
1490     case mul  :  __ z_msg(Z_tos, __ stackTop()); __ pop_l(); break;
1491     case _and :  __ z_ng(Z_tos,  __ stackTop()); __ pop_l(); break;
1492     case _or  :  __ z_og(Z_tos,  __ stackTop()); __ pop_l(); break;
1493     case _xor :  __ z_xg(Z_tos,  __ stackTop()); __ pop_l(); break;
1494     default   : ShouldNotReachHere(); break;
1495   }
1496   return;
1497 }
1498 
1499 // Common part of idiv/irem.
1500 static void idiv_helper(InterpreterMacroAssembler * _masm, address exception) {
1501   NearLabel not_null;
1502 
1503   // Use register pair Z_tmp_1, Z_tmp_2 for DIVIDE SINGLE.
1504   assert(Z_tmp_1->successor() == Z_tmp_2, " need even/odd register pair for idiv/irem");
1505 
1506   // Get dividend.
1507   __ pop_i(Z_tmp_2);
1508 
1509   // If divisor == 0 throw exception.
1510   __ compare32_and_branch(Z_tos, (intptr_t) 0,
1511                           Assembler::bcondNotEqual, not_null   );
1512   __ load_absolute_address(Z_R1_scratch, exception);
1513   __ z_br(Z_R1_scratch);
1514 
1515   __ bind(not_null);
1516 
1517   __ z_lgfr(Z_tmp_2, Z_tmp_2);   // Sign extend dividend.
1518   __ z_dsgfr(Z_tmp_1, Z_tos);    // Do it.
1519 }
1520 
1521 void TemplateTable::idiv() {
1522   transition(itos, itos);
1523 
1524   idiv_helper(_masm, Interpreter::_throw_ArithmeticException_entry);
1525   __ z_llgfr(Z_tos, Z_tmp_2);     // Result is in Z_tmp_2.
1526 }
1527 
1528 void TemplateTable::irem() {
1529   transition(itos, itos);
1530 
1531   idiv_helper(_masm, Interpreter::_throw_ArithmeticException_entry);
1532   __ z_llgfr(Z_tos, Z_tmp_1);     // Result is in Z_tmp_1.
1533 }
1534 
1535 void TemplateTable::lmul() {
1536   transition(ltos, ltos);
1537 
1538   // Multiply with memory operand.
1539   __ z_msg(Z_tos, __ stackTop());
1540   __ pop_l();  // Pop operand.
1541 }
1542 
1543 // Common part of ldiv/lrem.
1544 //
1545 // Input:
1546 //     Z_tos := the divisor (dividend still on stack)
1547 //
1548 // Updated registers:
1549 //     Z_tmp_1 := pop_l() % Z_tos     ; if is_ldiv == false
1550 //     Z_tmp_2 := pop_l() / Z_tos     ; if is_ldiv == true
1551 //
1552 static void ldiv_helper(InterpreterMacroAssembler * _masm, address exception, bool is_ldiv) {
1553   NearLabel not_null, done;
1554 
1555   // Use register pair Z_tmp_1, Z_tmp_2 for DIVIDE SINGLE.
1556   assert(Z_tmp_1->successor() == Z_tmp_2,
1557          " need even/odd register pair for idiv/irem");
1558 
1559   // Get dividend.
1560   __ pop_l(Z_tmp_2);
1561 
1562   // If divisor == 0 throw exception.
1563   __ compare64_and_branch(Z_tos, (intptr_t)0, Assembler::bcondNotEqual, not_null);
1564   __ load_absolute_address(Z_R1_scratch, exception);
1565   __ z_br(Z_R1_scratch);
1566 
1567   __ bind(not_null);
1568   // Special case for dividend == 0x8000 and divisor == -1.
1569   if (is_ldiv) {
1570     // result := Z_tmp_2 := - dividend
1571     __ z_lcgr(Z_tmp_2, Z_tmp_2);
1572   } else {
1573     // result remainder := Z_tmp_1 := 0
1574     __ clear_reg(Z_tmp_1, true, false);  // Don't set CC.
1575   }
1576 
1577   // if divisor == -1 goto done
1578   __ compare64_and_branch(Z_tos, -1, Assembler::bcondEqual, done);
1579   if (is_ldiv)
1580     // Restore sign, because divisor != -1.
1581     __ z_lcgr(Z_tmp_2, Z_tmp_2);
1582   __ z_dsgr(Z_tmp_1, Z_tos);    // Do it.
1583   __ bind(done);
1584 }
1585 
1586 void TemplateTable::ldiv() {
1587   transition(ltos, ltos);
1588 
1589   ldiv_helper(_masm, Interpreter::_throw_ArithmeticException_entry, true /*is_ldiv*/);
1590   __ z_lgr(Z_tos, Z_tmp_2);     // Result is in Z_tmp_2.
1591 }
1592 
1593 void TemplateTable::lrem() {
1594   transition(ltos, ltos);
1595 
1596   ldiv_helper(_masm, Interpreter::_throw_ArithmeticException_entry, false /*is_ldiv*/);
1597   __ z_lgr(Z_tos, Z_tmp_1);     // Result is in Z_tmp_1.
1598 }
1599 
1600 void TemplateTable::lshl() {
1601   transition(itos, ltos);
1602 
1603   // Z_tos: shift amount
1604   __ pop_l(Z_tmp_1);              // Get shift value.
1605   __ z_sllg(Z_tos, Z_tmp_1, 0, Z_tos);
1606 }
1607 
1608 void TemplateTable::lshr() {
1609   transition(itos, ltos);
1610 
1611   // Z_tos: shift amount
1612   __ pop_l(Z_tmp_1);              // Get shift value.
1613   __ z_srag(Z_tos, Z_tmp_1, 0, Z_tos);
1614 }
1615 
1616 void TemplateTable::lushr() {
1617   transition(itos, ltos);
1618 
1619   // Z_tos: shift amount
1620   __ pop_l(Z_tmp_1);              // Get shift value.
1621   __ z_srlg(Z_tos, Z_tmp_1, 0, Z_tos);
1622 }
1623 
1624 void TemplateTable::fop2(Operation op) {
1625   transition(ftos, ftos);
1626 
1627   switch (op) {
1628     case add:
1629       // Add memory operand.
1630       __ z_aeb(Z_ftos, __ stackTop()); __ pop_f(); return;
1631     case sub:
1632       // Sub memory operand.
1633       __ z_ler(Z_F1, Z_ftos);    // first operand
1634       __ pop_f(Z_ftos);          // second operand from stack
1635       __ z_sebr(Z_ftos, Z_F1);
1636       return;
1637     case mul:
1638       // Multiply with memory operand.
1639       __ z_meeb(Z_ftos, __ stackTop()); __ pop_f(); return;
1640     case div:
1641       __ z_ler(Z_F1, Z_ftos);    // first operand
1642       __ pop_f(Z_ftos);          // second operand from stack
1643       __ z_debr(Z_ftos, Z_F1);
1644       return;
1645     case rem:
1646       // Do runtime call.
1647       __ z_ler(Z_FARG2, Z_ftos);  // divisor
1648       __ pop_f(Z_FARG1);          // dividend
1649       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1650       // Result should be in the right place (Z_ftos == Z_FRET).
1651       return;
1652     default:
1653       ShouldNotReachHere();
1654       return;
1655   }
1656 }
1657 
1658 void TemplateTable::dop2(Operation op) {
1659   transition(dtos, dtos);
1660 
1661   switch (op) {
1662     case add:
1663       // Add memory operand.
1664       __ z_adb(Z_ftos, __ stackTop()); __ pop_d(); return;
1665     case sub:
1666       // Sub memory operand.
1667       __ z_ldr(Z_F1, Z_ftos);    // first operand
1668       __ pop_d(Z_ftos);          // second operand from stack
1669       __ z_sdbr(Z_ftos, Z_F1);
1670       return;
1671     case mul:
1672       // Multiply with memory operand.
1673       __ z_mdb(Z_ftos, __ stackTop()); __ pop_d(); return;
1674     case div:
1675       __ z_ldr(Z_F1, Z_ftos);    // first operand
1676       __ pop_d(Z_ftos);          // second operand from stack
1677       __ z_ddbr(Z_ftos, Z_F1);
1678       return;
1679     case rem:
1680       // Do runtime call.
1681       __ z_ldr(Z_FARG2, Z_ftos);  // divisor
1682       __ pop_d(Z_FARG1);          // dividend
1683       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1684       // Result should be in the right place (Z_ftos == Z_FRET).
1685       return;
1686     default:
1687       ShouldNotReachHere();
1688       return;
1689   }
1690 }
1691 
1692 void TemplateTable::ineg() {
1693   transition(itos, itos);
1694   __ z_lcr(Z_tos);
1695 }
1696 
1697 void TemplateTable::lneg() {
1698   transition(ltos, ltos);
1699   __ z_lcgr(Z_tos);
1700 }
1701 
1702 void TemplateTable::fneg() {
1703   transition(ftos, ftos);
1704   __ z_lcebr(Z_ftos, Z_ftos);
1705 }
1706 
1707 void TemplateTable::dneg() {
1708   transition(dtos, dtos);
1709   __ z_lcdbr(Z_ftos, Z_ftos);
1710 }
1711 
1712 void TemplateTable::iinc() {
1713   transition(vtos, vtos);
1714 
1715   Address local;
1716   __ z_lb(Z_R0_scratch, at_bcp(2)); // Get constant.
1717   locals_index(Z_R1_scratch);
1718   local = iaddress(_masm, Z_R1_scratch);
1719   __ z_a(Z_R0_scratch, local);
1720   __ reg2mem_opt(Z_R0_scratch, local, false);
1721 }
1722 
1723 void TemplateTable::wide_iinc() {
1724   transition(vtos, vtos);
1725 
1726   // Z_tmp_1 := increment
1727   __ get_2_byte_integer_at_bcp(Z_tmp_1, 4, InterpreterMacroAssembler::Signed);
1728   // Z_R1_scratch := index of local to increment
1729   locals_index_wide(Z_tmp_2);
1730   // Load, increment, and store.
1731   __ access_local_int(Z_tmp_2, Z_tos);
1732   __ z_agr(Z_tos,  Z_tmp_1);
1733   // Shifted index is still in Z_tmp_2.
1734   __ reg2mem_opt(Z_tos, Address(Z_locals, Z_tmp_2), false);
1735 }
1736 
1737 
1738 void TemplateTable::convert() {
1739   // Checking
1740 #ifdef ASSERT
1741   TosState   tos_in  = ilgl;
1742   TosState   tos_out = ilgl;
1743 
1744   switch (bytecode()) {
1745     case Bytecodes::_i2l:
1746     case Bytecodes::_i2f:
1747     case Bytecodes::_i2d:
1748     case Bytecodes::_i2b:
1749     case Bytecodes::_i2c:
1750     case Bytecodes::_i2s:
1751       tos_in = itos;
1752       break;
1753     case Bytecodes::_l2i:
1754     case Bytecodes::_l2f:
1755     case Bytecodes::_l2d:
1756       tos_in = ltos;
1757       break;
1758     case Bytecodes::_f2i:
1759     case Bytecodes::_f2l:
1760     case Bytecodes::_f2d:
1761       tos_in = ftos;
1762       break;
1763     case Bytecodes::_d2i:
1764     case Bytecodes::_d2l:
1765     case Bytecodes::_d2f:
1766       tos_in = dtos;
1767       break;
1768     default :
1769       ShouldNotReachHere();
1770   }
1771   switch (bytecode()) {
1772     case Bytecodes::_l2i:
1773     case Bytecodes::_f2i:
1774     case Bytecodes::_d2i:
1775     case Bytecodes::_i2b:
1776     case Bytecodes::_i2c:
1777     case Bytecodes::_i2s:
1778       tos_out = itos;
1779       break;
1780     case Bytecodes::_i2l:
1781     case Bytecodes::_f2l:
1782     case Bytecodes::_d2l:
1783       tos_out = ltos;
1784       break;
1785     case Bytecodes::_i2f:
1786     case Bytecodes::_l2f:
1787     case Bytecodes::_d2f:
1788       tos_out = ftos;
1789       break;
1790     case Bytecodes::_i2d:
1791     case Bytecodes::_l2d:
1792     case Bytecodes::_f2d:
1793       tos_out = dtos;
1794       break;
1795     default :
1796       ShouldNotReachHere();
1797   }
1798 
1799   transition(tos_in, tos_out);
1800 #endif // ASSERT
1801 
1802   // Conversion
1803   Label done;
1804   switch (bytecode()) {
1805     case Bytecodes::_i2l:
1806       __ z_lgfr(Z_tos, Z_tos);
1807       return;
1808     case Bytecodes::_i2f:
1809       __ z_cefbr(Z_ftos, Z_tos);
1810       return;
1811     case Bytecodes::_i2d:
1812       __ z_cdfbr(Z_ftos, Z_tos);
1813       return;
1814     case Bytecodes::_i2b:
1815       // Sign extend least significant byte.
1816       __ move_reg_if_needed(Z_tos, T_BYTE, Z_tos, T_INT);
1817       return;
1818     case Bytecodes::_i2c:
1819       // Zero extend 2 least significant bytes.
1820       __ move_reg_if_needed(Z_tos, T_CHAR, Z_tos, T_INT);
1821       return;
1822     case Bytecodes::_i2s:
1823       // Sign extend 2 least significant bytes.
1824       __ move_reg_if_needed(Z_tos, T_SHORT, Z_tos, T_INT);
1825       return;
1826     case Bytecodes::_l2i:
1827       // Sign-extend not needed here, upper 4 bytes of int value in register are ignored.
1828       return;
1829     case Bytecodes::_l2f:
1830       __ z_cegbr(Z_ftos, Z_tos);
1831       return;
1832     case Bytecodes::_l2d:
1833       __ z_cdgbr(Z_ftos, Z_tos);
1834       return;
1835     case Bytecodes::_f2i:
1836     case Bytecodes::_f2l:
1837       __ clear_reg(Z_tos, true, false);  // Don't set CC.
1838       __ z_cebr(Z_ftos, Z_ftos);
1839       __ z_brno(done); // NaN -> 0
1840       if (bytecode() == Bytecodes::_f2i)
1841         __ z_cfebr(Z_tos, Z_ftos, Assembler::to_zero);
1842       else // bytecode() == Bytecodes::_f2l
1843         __ z_cgebr(Z_tos, Z_ftos, Assembler::to_zero);
1844       break;
1845     case Bytecodes::_f2d:
1846       __ move_freg_if_needed(Z_ftos, T_DOUBLE, Z_ftos, T_FLOAT);
1847       return;
1848     case Bytecodes::_d2i:
1849     case Bytecodes::_d2l:
1850       __ clear_reg(Z_tos, true, false);  // Ddon't set CC.
1851       __ z_cdbr(Z_ftos, Z_ftos);
1852       __ z_brno(done); // NaN -> 0
1853       if (bytecode() == Bytecodes::_d2i)
1854         __ z_cfdbr(Z_tos, Z_ftos, Assembler::to_zero);
1855       else // Bytecodes::_d2l
1856         __ z_cgdbr(Z_tos, Z_ftos, Assembler::to_zero);
1857       break;
1858     case Bytecodes::_d2f:
1859       __ move_freg_if_needed(Z_ftos, T_FLOAT, Z_ftos, T_DOUBLE);
1860       return;
1861     default:
1862       ShouldNotReachHere();
1863   }
1864   __ bind(done);
1865 }
1866 
1867 void TemplateTable::lcmp() {
1868   transition(ltos, itos);
1869 
1870   Label   done;
1871   Register val1 = Z_R0_scratch;
1872   Register val2 = Z_R1_scratch;
1873 
1874   if (VM_Version::has_LoadStoreConditional()) {
1875     __ pop_l(val1);           // pop value 1.
1876     __ z_lghi(val2,  -1);     // lt value
1877     __ z_cgr(val1, Z_tos);    // Compare with Z_tos (value 2). Protect CC under all circumstances.
1878     __ z_lghi(val1,   1);     // gt value
1879     __ z_lghi(Z_tos,  0);     // eq value
1880 
1881     __ z_locgr(Z_tos, val1, Assembler::bcondHigh);
1882     __ z_locgr(Z_tos, val2, Assembler::bcondLow);
1883   } else {
1884     __ pop_l(val1);           // Pop value 1.
1885     __ z_cgr(val1, Z_tos);    // Compare with Z_tos (value 2). Protect CC under all circumstances.
1886 
1887     __ z_lghi(Z_tos,  0);     // eq value
1888     __ z_bre(done);
1889 
1890     __ z_lghi(Z_tos,  1);     // gt value
1891     __ z_brh(done);
1892 
1893     __ z_lghi(Z_tos, -1);     // lt value
1894   }
1895 
1896   __ bind(done);
1897 }
1898 
1899 
1900 void TemplateTable::float_cmp(bool is_float, int unordered_result) {
1901   Label done;
1902 
1903   if (is_float) {
1904     __ pop_f(Z_FARG2);
1905     __ z_cebr(Z_FARG2, Z_ftos);
1906   } else {
1907     __ pop_d(Z_FARG2);
1908     __ z_cdbr(Z_FARG2, Z_ftos);
1909   }
1910 
1911   if (VM_Version::has_LoadStoreConditional()) {
1912     Register one       = Z_R0_scratch;
1913     Register minus_one = Z_R1_scratch;
1914     __ z_lghi(minus_one,  -1);
1915     __ z_lghi(one,  1);
1916     __ z_lghi(Z_tos, 0);
1917     __ z_locgr(Z_tos, one,       unordered_result == 1 ? Assembler::bcondHighOrNotOrdered : Assembler::bcondHigh);
1918     __ z_locgr(Z_tos, minus_one, unordered_result == 1 ? Assembler::bcondLow              : Assembler::bcondLowOrNotOrdered);
1919   } else {
1920     // Z_FARG2 == Z_ftos
1921     __ clear_reg(Z_tos, false, false);
1922     __ z_bre(done);
1923 
1924     // F_ARG2 > Z_Ftos, or unordered
1925     __ z_lhi(Z_tos, 1);
1926     __ z_brc(unordered_result == 1 ? Assembler::bcondHighOrNotOrdered : Assembler::bcondHigh, done);
1927 
1928     // F_ARG2 < Z_FTOS, or unordered
1929     __ z_lhi(Z_tos, -1);
1930 
1931     __ bind(done);
1932   }
1933 }
1934 
1935 void TemplateTable::branch(bool is_jsr, bool is_wide) {
1936   const Register   bumped_count = Z_tmp_1;
1937   const Register   method       = Z_tmp_2;
1938   const Register   m_counters   = Z_R1_scratch;
1939   const Register   mdo          = Z_tos;
1940 
1941   BLOCK_COMMENT("TemplateTable::branch {");
1942   __ get_method(method);
1943   __ profile_taken_branch(mdo, bumped_count);
1944 
1945   const ByteSize ctr_offset = InvocationCounter::counter_offset();
1946   const ByteSize be_offset  = MethodCounters::backedge_counter_offset()   + ctr_offset;
1947   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() + ctr_offset;
1948 
1949   // Get (wide) offset to disp.
1950   const Register disp = Z_ARG5;
1951   if (is_wide) {
1952     __ get_4_byte_integer_at_bcp(disp, 1);
1953   } else {
1954     __ get_2_byte_integer_at_bcp(disp, 1, InterpreterMacroAssembler::Signed);
1955   }
1956 
1957   // Handle all the JSR stuff here, then exit.
1958   // It's much shorter and cleaner than intermingling with the
1959   // non-JSR normal-branch stuff occurring below.
1960   if (is_jsr) {
1961     // Compute return address as bci in Z_tos.
1962     __ z_lgr(Z_R1_scratch, Z_bcp);
1963     __ z_sg(Z_R1_scratch, Address(method, Method::const_offset()));
1964     __ add2reg(Z_tos, (is_wide ? 5 : 3) - in_bytes(ConstMethod::codes_offset()), Z_R1_scratch);
1965 
1966     // Bump bcp to target of JSR.
1967     __ z_agr(Z_bcp, disp);
1968     // Push return address for "ret" on stack.
1969     __ push_ptr(Z_tos);
1970     // And away we go!
1971     __ dispatch_next(vtos, 0 , true);
1972     return;
1973   }
1974 
1975   // Normal (non-jsr) branch handling.
1976 
1977   // Bump bytecode pointer by displacement (take the branch).
1978   __ z_agr(Z_bcp, disp);
1979 
1980   assert(UseLoopCounter || !UseOnStackReplacement,
1981          "on-stack-replacement requires loop counters");
1982 
1983   NearLabel backedge_counter_overflow;
1984   NearLabel profile_method;
1985   NearLabel dispatch;
1986   int       increment = InvocationCounter::count_increment;
1987 
1988   if (UseLoopCounter) {
1989     // Increment backedge counter for backward branches.
1990     // disp: target offset
1991     // Z_bcp: target bcp
1992     // Z_locals: locals pointer
1993     //
1994     // Count only if backward branch.
1995     __ compare32_and_branch(disp, (intptr_t)0, Assembler::bcondHigh, dispatch);
1996 
1997     if (TieredCompilation) {
1998       Label noCounters;
1999 
2000       if (ProfileInterpreter) {
2001         NearLabel   no_mdo;
2002 
2003         // Are we profiling?
2004         __ load_and_test_long(mdo, Address(method, Method::method_data_offset()));
2005         __ branch_optimized(Assembler::bcondZero, no_mdo);
2006 
2007         // Increment the MDO backedge counter.
2008         const Address mdo_backedge_counter(mdo, MethodData::backedge_counter_offset() + InvocationCounter::counter_offset());
2009 
2010         const Address mask(mdo, MethodData::backedge_mask_offset());
2011         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
2012                                    Z_ARG2, false, Assembler::bcondZero,
2013                                    UseOnStackReplacement ? &backedge_counter_overflow : NULL);
2014         __ z_bru(dispatch);
2015         __ bind(no_mdo);
2016       }
2017 
2018       // Increment backedge counter in MethodCounters*.
2019       __ get_method_counters(method, m_counters, noCounters);
2020       const Address mask(m_counters, MethodCounters::backedge_mask_offset());
2021       __ increment_mask_and_jump(Address(m_counters, be_offset),
2022                                  increment, mask,
2023                                  Z_ARG2, false, Assembler::bcondZero,
2024                                  UseOnStackReplacement ? &backedge_counter_overflow : NULL);
2025       __ bind(noCounters);
2026     } else {
2027       Register counter = Z_tos;
2028       Label    noCounters;
2029       // Get address of MethodCounters object.
2030       __ get_method_counters(method, m_counters, noCounters);
2031       // Increment backedge counter.
2032       __ increment_backedge_counter(m_counters, counter);
2033 
2034       if (ProfileInterpreter) {
2035         // Test to see if we should create a method data obj.
2036         __ z_cl(counter, Address(m_counters, MethodCounters::interpreter_profile_limit_offset()));
2037         __ z_brl(dispatch);
2038 
2039         // If no method data exists, go to profile method.
2040         __ test_method_data_pointer(Z_ARG4/*result unused*/, profile_method);
2041 
2042         if (UseOnStackReplacement) {
2043           // Check for overflow against 'bumped_count' which is the MDO taken count.
2044           __ z_cl(bumped_count, Address(m_counters, MethodCounters::interpreter_backward_branch_limit_offset()));
2045           __ z_brl(dispatch);
2046 
2047           // When ProfileInterpreter is on, the backedge_count comes
2048           // from the methodDataOop, which value does not get reset on
2049           // the call to frequency_counter_overflow(). To avoid
2050           // excessive calls to the overflow routine while the method is
2051           // being compiled, add a second test to make sure the overflow
2052           // function is called only once every overflow_frequency.
2053           const int overflow_frequency = 1024;
2054           __ and_imm(bumped_count, overflow_frequency - 1);
2055           __ z_brz(backedge_counter_overflow);
2056 
2057         }
2058       } else {
2059         if (UseOnStackReplacement) {
2060           // Check for overflow against 'counter', which is the sum of the
2061           // counters.
2062           __ z_cl(counter, Address(m_counters, MethodCounters::interpreter_backward_branch_limit_offset()));
2063           __ z_brh(backedge_counter_overflow);
2064         }
2065       }
2066       __ bind(noCounters);
2067     }
2068 
2069     __ bind(dispatch);
2070   }
2071 
2072   // Pre-load the next target bytecode into rbx.
2073   __ z_llgc(Z_bytecode, Address(Z_bcp, (intptr_t) 0));
2074 
2075   // Continue with the bytecode @ target.
2076   // Z_tos: Return bci for jsr's, unused otherwise.
2077   // Z_bytecode: target bytecode
2078   // Z_bcp: target bcp
2079   __ dispatch_only(vtos, true);
2080 
2081   // Out-of-line code runtime calls.
2082   if (UseLoopCounter) {
2083     if (ProfileInterpreter) {
2084       // Out-of-line code to allocate method data oop.
2085       __ bind(profile_method);
2086 
2087       __ call_VM(noreg,
2088                  CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
2089       __ z_llgc(Z_bytecode, Address(Z_bcp, (intptr_t) 0));  // Restore target bytecode.
2090       __ set_method_data_pointer_for_bcp();
2091       __ z_bru(dispatch);
2092     }
2093 
2094     if (UseOnStackReplacement) {
2095 
2096       // invocation counter overflow
2097       __ bind(backedge_counter_overflow);
2098 
2099       __ z_lcgr(Z_ARG2, disp); // Z_ARG2 := -disp
2100       __ z_agr(Z_ARG2, Z_bcp); // Z_ARG2 := branch target bcp - disp == branch bcp
2101       __ call_VM(noreg,
2102                  CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow),
2103                  Z_ARG2);
2104 
2105       // Z_RET: osr nmethod (osr ok) or NULL (osr not possible).
2106       __ compare64_and_branch(Z_RET, (intptr_t) 0, Assembler::bcondEqual, dispatch);
2107 
2108       // Nmethod may have been invalidated (VM may block upon call_VM return).
2109       __ z_cliy(nmethod::state_offset(), Z_RET, nmethod::in_use);
2110       __ z_brne(dispatch);
2111 
2112       // Migrate the interpreter frame off of the stack.
2113 
2114       __ z_lgr(Z_tmp_1, Z_RET); // Save the nmethod.
2115 
2116       call_VM(noreg,
2117               CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2118 
2119       // Z_RET is OSR buffer, move it to expected parameter location.
2120       __ lgr_if_needed(Z_ARG1, Z_RET);
2121 
2122       // Pop the interpreter frame ...
2123       __ pop_interpreter_frame(Z_R14, Z_ARG2/*tmp1*/, Z_ARG3/*tmp2*/);
2124 
2125       // ... and begin the OSR nmethod.
2126       __ z_lg(Z_R1_scratch, Address(Z_tmp_1, nmethod::osr_entry_point_offset()));
2127       __ z_br(Z_R1_scratch);
2128     }
2129   }
2130   BLOCK_COMMENT("} TemplateTable::branch");
2131 }
2132 
2133 void TemplateTable::if_0cmp(Condition cc) {
2134   transition(itos, vtos);
2135 
2136   // Assume branch is more often taken than not (loops use backward branches).
2137   NearLabel not_taken;
2138   __ compare32_and_branch(Z_tos, (intptr_t) 0, j_not(cc), not_taken);
2139   branch(false, false);
2140   __ bind(not_taken);
2141   __ profile_not_taken_branch(Z_tos);
2142 }
2143 
2144 void TemplateTable::if_icmp(Condition cc) {
2145   transition(itos, vtos);
2146 
2147   // Assume branch is more often taken than not (loops use backward branches).
2148   NearLabel not_taken;
2149   __ pop_i(Z_R0_scratch);
2150   __ compare32_and_branch(Z_R0_scratch, Z_tos, j_not(cc), not_taken);
2151   branch(false, false);
2152   __ bind(not_taken);
2153   __ profile_not_taken_branch(Z_tos);
2154 }
2155 
2156 void TemplateTable::if_nullcmp(Condition cc) {
2157   transition(atos, vtos);
2158 
2159   // Assume branch is more often taken than not (loops use backward branches) .
2160   NearLabel not_taken;
2161   __ compare64_and_branch(Z_tos, (intptr_t) 0, j_not(cc), not_taken);
2162   branch(false, false);
2163   __ bind(not_taken);
2164   __ profile_not_taken_branch(Z_tos);
2165 }
2166 
2167 void TemplateTable::if_acmp(Condition cc) {
2168   transition(atos, vtos);
2169   // Assume branch is more often taken than not (loops use backward branches).
2170   NearLabel not_taken;
2171   __ pop_ptr(Z_ARG2);
2172   __ verify_oop(Z_ARG2);
2173   __ verify_oop(Z_tos);
2174   __ compareU64_and_branch(Z_tos, Z_ARG2, j_not(cc), not_taken);
2175   branch(false, false);
2176   __ bind(not_taken);
2177   __ profile_not_taken_branch(Z_ARG3);
2178 }
2179 
2180 void TemplateTable::ret() {
2181   transition(vtos, vtos);
2182 
2183   locals_index(Z_tmp_1);
2184   // Get return bci, compute return bcp. Must load 64 bits.
2185   __ mem2reg_opt(Z_tmp_1, iaddress(_masm, Z_tmp_1));
2186   __ profile_ret(Z_tmp_1, Z_tmp_2);
2187   __ get_method(Z_tos);
2188   __ mem2reg_opt(Z_R1_scratch, Address(Z_tos, Method::const_offset()));
2189   __ load_address(Z_bcp, Address(Z_R1_scratch, Z_tmp_1, ConstMethod::codes_offset()));
2190   __ dispatch_next(vtos, 0 , true);
2191 }
2192 
2193 void TemplateTable::wide_ret() {
2194   transition(vtos, vtos);
2195 
2196   locals_index_wide(Z_tmp_1);
2197   // Get return bci, compute return bcp.
2198   __ mem2reg_opt(Z_tmp_1, aaddress(_masm, Z_tmp_1));
2199   __ profile_ret(Z_tmp_1, Z_tmp_2);
2200   __ get_method(Z_tos);
2201   __ mem2reg_opt(Z_R1_scratch, Address(Z_tos, Method::const_offset()));
2202   __ load_address(Z_bcp, Address(Z_R1_scratch, Z_tmp_1, ConstMethod::codes_offset()));
2203   __ dispatch_next(vtos, 0, true);
2204 }
2205 
2206 void TemplateTable::tableswitch () {
2207   transition(itos, vtos);
2208 
2209   NearLabel default_case, continue_execution;
2210   Register  bcp = Z_ARG5;
2211   // Align bcp.
2212   __ load_address(bcp, at_bcp(BytesPerInt));
2213   __ z_nill(bcp, (-BytesPerInt) & 0xffff);
2214 
2215   // Load lo & hi.
2216   Register low  = Z_tmp_1;
2217   Register high = Z_tmp_2;
2218 
2219   // Load low into 64 bits, since used for address calculation.
2220   __ mem2reg_signed_opt(low, Address(bcp, BytesPerInt));
2221   __ mem2reg_opt(high, Address(bcp, 2 * BytesPerInt), false);
2222   // Sign extend "label" value for address calculation.
2223   __ z_lgfr(Z_tos, Z_tos);
2224 
2225   // Check against lo & hi.
2226   __ compare32_and_branch(Z_tos, low, Assembler::bcondLow, default_case);
2227   __ compare32_and_branch(Z_tos, high, Assembler::bcondHigh, default_case);
2228 
2229   // Lookup dispatch offset.
2230   __ z_sgr(Z_tos, low);
2231   Register jump_table_offset = Z_ARG3;
2232   // Index2offset; index in Z_tos is killed by profile_switch_case.
2233   __ z_sllg(jump_table_offset, Z_tos, LogBytesPerInt);
2234   __ profile_switch_case(Z_tos, Z_ARG4 /*tmp for mdp*/, low/*tmp*/, Z_bytecode/*tmp*/);
2235 
2236   Register index = Z_tmp_2;
2237 
2238   // Load index sign extended for addressing.
2239   __ mem2reg_signed_opt(index, Address(bcp, jump_table_offset, 3 * BytesPerInt));
2240 
2241   // Continue execution.
2242   __ bind(continue_execution);
2243 
2244   // Load next bytecode.
2245   __ z_llgc(Z_bytecode, Address(Z_bcp, index));
2246   __ z_agr(Z_bcp, index); // Advance bcp.
2247   __ dispatch_only(vtos, true);
2248 
2249   // Handle default.
2250   __ bind(default_case);
2251 
2252   __ profile_switch_default(Z_tos);
2253   __ mem2reg_signed_opt(index, Address(bcp));
2254   __ z_bru(continue_execution);
2255 }
2256 
2257 void TemplateTable::lookupswitch () {
2258   transition(itos, itos);
2259   __ stop("lookupswitch bytecode should have been rewritten");
2260 }
2261 
2262 void TemplateTable::fast_linearswitch () {
2263   transition(itos, vtos);
2264 
2265   Label    loop_entry, loop, found, continue_execution;
2266   Register bcp = Z_ARG5;
2267 
2268   // Align bcp.
2269   __ load_address(bcp, at_bcp(BytesPerInt));
2270   __ z_nill(bcp, (-BytesPerInt) & 0xffff);
2271 
2272   // Start search with last case.
2273   Register current_case_offset = Z_tmp_1;
2274 
2275   __ mem2reg_signed_opt(current_case_offset, Address(bcp, BytesPerInt));
2276   __ z_sllg(current_case_offset, current_case_offset, LogBytesPerWord);   // index2bytes
2277   __ z_bru(loop_entry);
2278 
2279   // table search
2280   __ bind(loop);
2281 
2282   __ z_c(Z_tos, Address(bcp, current_case_offset, 2 * BytesPerInt));
2283   __ z_bre(found);
2284 
2285   __ bind(loop_entry);
2286   __ z_aghi(current_case_offset, -2 * BytesPerInt);  // Decrement.
2287   __ z_brnl(loop);
2288 
2289   // default case
2290   Register   offset = Z_tmp_2;
2291 
2292   __ profile_switch_default(Z_tos);
2293   // Load offset sign extended for addressing.
2294   __ mem2reg_signed_opt(offset, Address(bcp));
2295   __ z_bru(continue_execution);
2296 
2297   // Entry found -> get offset.
2298   __ bind(found);
2299   __ mem2reg_signed_opt(offset, Address(bcp, current_case_offset, 3 * BytesPerInt));
2300   // Profile that this case was taken.
2301   Register current_case_idx = Z_ARG4;
2302   __ z_srlg(current_case_idx, current_case_offset, LogBytesPerWord); // bytes2index
2303   __ profile_switch_case(current_case_idx, Z_tos, bcp, Z_bytecode);
2304 
2305   // Continue execution.
2306   __ bind(continue_execution);
2307 
2308   // Load next bytecode.
2309   __ z_llgc(Z_bytecode, Address(Z_bcp, offset, 0));
2310   __ z_agr(Z_bcp, offset); // Advance bcp.
2311   __ dispatch_only(vtos, true);
2312 }
2313 
2314 
2315 void TemplateTable::fast_binaryswitch() {
2316 
2317   transition(itos, vtos);
2318 
2319   // Implementation using the following core algorithm:
2320   //
2321   // int binary_search(int key, LookupswitchPair* array, int n) {
2322   //   // Binary search according to "Methodik des Programmierens" by
2323   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2324   //   int i = 0;
2325   //   int j = n;
2326   //   while (i+1 < j) {
2327   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2328   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2329   //     // where a stands for the array and assuming that the (inexisting)
2330   //     // element a[n] is infinitely big.
2331   //     int h = (i + j) >> 1;
2332   //     // i < h < j
2333   //     if (key < array[h].fast_match()) {
2334   //       j = h;
2335   //     } else {
2336   //       i = h;
2337   //     }
2338   //   }
2339   //   // R: a[i] <= key < a[i+1] or Q
2340   //   // (i.e., if key is within array, i is the correct index)
2341   //   return i;
2342   // }
2343 
2344   // Register allocation
2345   // Note: Since we use the indices in address operands, we do all the
2346   // computation in 64 bits.
2347   const Register key   = Z_tos; // Already set (tosca).
2348   const Register array = Z_tmp_1;
2349   const Register i     = Z_tmp_2;
2350   const Register j     = Z_ARG5;
2351   const Register h     = Z_ARG4;
2352   const Register temp  = Z_R1_scratch;
2353 
2354   // Find array start.
2355   __ load_address(array, at_bcp(3 * BytesPerInt));
2356   __ z_nill(array, (-BytesPerInt) & 0xffff);   // align
2357 
2358   // Initialize i & j.
2359   __ clear_reg(i, true, false);  // i = 0;  Don't set CC.
2360   __ mem2reg_signed_opt(j, Address(array, -BytesPerInt)); // j = length(array);
2361 
2362   // And start.
2363   Label entry;
2364   __ z_bru(entry);
2365 
2366   // binary search loop
2367   {
2368     NearLabel   loop;
2369 
2370     __ bind(loop);
2371 
2372     // int h = (i + j) >> 1;
2373     __ add2reg_with_index(h, 0, i, j); // h = i + j;
2374     __ z_srag(h, h, 1);                // h = (i + j) >> 1;
2375 
2376     // if (key < array[h].fast_match()) {
2377     //   j = h;
2378     // } else {
2379     //   i = h;
2380     // }
2381 
2382     // Convert array[h].match to native byte-ordering before compare.
2383     __ z_sllg(temp, h, LogBytesPerWord);   // index2bytes
2384     __ mem2reg_opt(temp, Address(array, temp), false);
2385 
2386     NearLabel  else_;
2387 
2388     __ compare32_and_branch(key, temp, Assembler::bcondNotLow, else_);
2389     // j = h if (key <  array[h].fast_match())
2390     __ z_lgr(j, h);
2391     __ z_bru(entry); // continue
2392 
2393     __ bind(else_);
2394 
2395     // i = h if (key >= array[h].fast_match())
2396     __ z_lgr(i, h);  // and fallthrough
2397 
2398     // while (i+1 < j)
2399     __ bind(entry);
2400 
2401     // if (i + 1 < j) continue search
2402     __ add2reg(h, 1, i);
2403     __ compare64_and_branch(h, j, Assembler::bcondLow, loop);
2404   }
2405 
2406   // End of binary search, result index is i (must check again!).
2407   NearLabel default_case;
2408 
2409   // h is no longer needed, so use it to hold the byte offset.
2410   __ z_sllg(h, i, LogBytesPerWord);   // index2bytes
2411   __ mem2reg_opt(temp, Address(array, h), false);
2412   __ compare32_and_branch(key, temp, Assembler::bcondNotEqual, default_case);
2413 
2414   // entry found -> j = offset
2415   __ mem2reg_signed_opt(j, Address(array, h, BytesPerInt));
2416   __ profile_switch_case(i, key, array, Z_bytecode);
2417   // Load next bytecode.
2418   __ z_llgc(Z_bytecode, Address(Z_bcp, j));
2419   __ z_agr(Z_bcp, j);       // Advance bcp.
2420   __ dispatch_only(vtos, true);
2421 
2422   // default case -> j = default offset
2423   __ bind(default_case);
2424 
2425   __ profile_switch_default(i);
2426   __ mem2reg_signed_opt(j, Address(array, -2 * BytesPerInt));
2427   // Load next bytecode.
2428   __ z_llgc(Z_bytecode, Address(Z_bcp, j));
2429   __ z_agr(Z_bcp, j);       // Advance bcp.
2430   __ dispatch_only(vtos, true);
2431 }
2432 
2433 void TemplateTable::_return(TosState state) {
2434   transition(state, state);
2435   assert(_desc->calls_vm(),
2436          "inconsistent calls_vm information"); // call in remove_activation
2437 
2438   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2439     Register Rthis  = Z_ARG2;
2440     Register Rklass = Z_ARG5;
2441     Label skip_register_finalizer;
2442     assert(state == vtos, "only valid state");
2443     __ z_lg(Rthis, aaddress(0));
2444     __ load_klass(Rklass, Rthis);
2445     __ testbit(Address(Rklass, Klass::access_flags_offset()), exact_log2(JVM_ACC_HAS_FINALIZER));
2446     __ z_bfalse(skip_register_finalizer);
2447     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), Rthis);
2448     __ bind(skip_register_finalizer);
2449   }
2450 
2451   if (SafepointMechanism::uses_thread_local_poll() && _desc->bytecode() != Bytecodes::_return_register_finalizer) {
2452     Label no_safepoint;
2453     const Address poll_byte_addr(Z_thread, in_bytes(Thread::polling_page_offset()) + 7 /* Big Endian */);
2454     __ z_tm(poll_byte_addr, SafepointMechanism::poll_bit());
2455     __ z_braz(no_safepoint);
2456     __ push(state);
2457     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::at_safepoint));
2458     __ pop(state);
2459     __ bind(no_safepoint);
2460   }
2461 
2462   if (state == itos) {
2463     // Narrow result if state is itos but result type is smaller.
2464     // Need to narrow in the return bytecode rather than in generate_return_entry
2465     // since compiled code callers expect the result to already be narrowed.
2466     __ narrow(Z_tos, Z_tmp_1); /* fall through */
2467   }
2468 
2469   __ remove_activation(state, Z_R14);
2470   __ z_br(Z_R14);
2471 }
2472 
2473 // ----------------------------------------------------------------------------
2474 // NOTE: Cpe_offset is already computed as byte offset, so we must not
2475 // shift it afterwards!
2476 void TemplateTable::resolve_cache_and_index(int byte_no,
2477                                             Register Rcache,
2478                                             Register cpe_offset,
2479                                             size_t index_size) {
2480   BLOCK_COMMENT("resolve_cache_and_index {");
2481   NearLabel      resolved;
2482   const Register bytecode_in_cpcache = Z_R1_scratch;
2483   const int      total_f1_offset = in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset());
2484   assert_different_registers(Rcache, cpe_offset, bytecode_in_cpcache);
2485 
2486   Bytecodes::Code code = bytecode();
2487   switch (code) {
2488     case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2489     case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2490   }
2491 
2492   {
2493     assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2494     __ get_cache_and_index_and_bytecode_at_bcp(Rcache, cpe_offset, bytecode_in_cpcache, byte_no, 1, index_size);
2495     // Have we resolved this bytecode?
2496     __ compare32_and_branch(bytecode_in_cpcache, (int)code, Assembler::bcondEqual, resolved);
2497   }
2498 
2499   // Resolve first time through.
2500   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2501   __ load_const_optimized(Z_ARG2, (int) code);
2502   __ call_VM(noreg, entry, Z_ARG2);
2503 
2504   // Update registers with resolved info.
2505   __ get_cache_and_index_at_bcp(Rcache, cpe_offset, 1, index_size);
2506   __ bind(resolved);
2507   BLOCK_COMMENT("} resolve_cache_and_index");
2508 }
2509 
2510 // The Rcache and index registers must be set before call.
2511 // Index is already a byte offset, don't shift!
2512 void TemplateTable::load_field_cp_cache_entry(Register obj,
2513                                               Register cache,
2514                                               Register index,
2515                                               Register off,
2516                                               Register flags,
2517                                               bool is_static = false) {
2518   assert_different_registers(cache, index, flags, off);
2519   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2520 
2521   // Field offset
2522   __ mem2reg_opt(off, Address(cache, index, cp_base_offset + ConstantPoolCacheEntry::f2_offset()));
2523   // Flags. Must load 64 bits.
2524   __ mem2reg_opt(flags, Address(cache, index, cp_base_offset + ConstantPoolCacheEntry::flags_offset()));
2525 
2526   // klass overwrite register
2527   if (is_static) {
2528     __ mem2reg_opt(obj, Address(cache, index, cp_base_offset + ConstantPoolCacheEntry::f1_offset()));
2529     __ mem2reg_opt(obj, Address(obj, Klass::java_mirror_offset()));
2530     __ resolve_oop_handle(obj);
2531   }
2532 }
2533 
2534 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2535                                                Register method,
2536                                                Register itable_index,
2537                                                Register flags,
2538                                                bool is_invokevirtual,
2539                                                bool is_invokevfinal, // unused
2540                                                bool is_invokedynamic) {
2541   BLOCK_COMMENT("load_invoke_cp_cache_entry {");
2542   // Setup registers.
2543   const Register cache     = Z_ARG1;
2544   const Register cpe_offset= flags;
2545   const ByteSize base_off  = ConstantPoolCache::base_offset();
2546   const ByteSize f1_off    = ConstantPoolCacheEntry::f1_offset();
2547   const ByteSize f2_off    = ConstantPoolCacheEntry::f2_offset();
2548   const ByteSize flags_off = ConstantPoolCacheEntry::flags_offset();
2549   const int method_offset  = in_bytes(base_off + ((byte_no == f2_byte) ? f2_off : f1_off));
2550   const int flags_offset   = in_bytes(base_off + flags_off);
2551   // Access constant pool cache fields.
2552   const int index_offset   = in_bytes(base_off + f2_off);
2553 
2554   assert_different_registers(method, itable_index, flags, cache);
2555   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2556 
2557   if (is_invokevfinal) {
2558     // Already resolved.
2559      assert(itable_index == noreg, "register not used");
2560      __ get_cache_and_index_at_bcp(cache, cpe_offset, 1);
2561   } else {
2562     // Need to resolve.
2563     resolve_cache_and_index(byte_no, cache, cpe_offset, is_invokedynamic ? sizeof(u4) : sizeof(u2));
2564   }
2565   __ z_lg(method, Address(cache, cpe_offset, method_offset));
2566 
2567   if (itable_index != noreg) {
2568     __ z_lg(itable_index, Address(cache, cpe_offset, index_offset));
2569   }
2570 
2571   // Only load the lower 4 bytes and fill high bytes of flags with zeros.
2572   // Callers depend on this zero-extension!!!
2573   // Attention: overwrites cpe_offset == flags
2574   __ z_llgf(flags, Address(cache, cpe_offset, flags_offset + (BytesPerLong-BytesPerInt)));
2575 
2576   BLOCK_COMMENT("} load_invoke_cp_cache_entry");
2577 }
2578 
2579 // The registers cache and index expected to be set before call.
2580 // Correct values of the cache and index registers are preserved.
2581 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2582                                             bool is_static, bool has_tos) {
2583 
2584   // Do the JVMTI work here to avoid disturbing the register state below.
2585   // We use c_rarg registers here because we want to use the register used in
2586   // the call to the VM
2587   if (!JvmtiExport::can_post_field_access()) {
2588     return;
2589   }
2590 
2591   // Check to see if a field access watch has been set before we
2592   // take the time to call into the VM.
2593   Label exit;
2594   assert_different_registers(cache, index, Z_tos);
2595   __ load_absolute_address(Z_tos, (address)JvmtiExport::get_field_access_count_addr());
2596   __ load_and_test_int(Z_R0, Address(Z_tos));
2597   __ z_brz(exit);
2598 
2599   // Index is returned as byte offset, do not shift!
2600   __ get_cache_and_index_at_bcp(Z_ARG3, Z_R1_scratch, 1);
2601 
2602   // cache entry pointer
2603   __ add2reg_with_index(Z_ARG3,
2604                         in_bytes(ConstantPoolCache::base_offset()),
2605                         Z_ARG3, Z_R1_scratch);
2606 
2607   if (is_static) {
2608     __ clear_reg(Z_ARG2, true, false); // NULL object reference. Don't set CC.
2609   } else {
2610     __ mem2reg_opt(Z_ARG2, at_tos());  // Get object pointer without popping it.
2611     __ verify_oop(Z_ARG2);
2612   }
2613   // Z_ARG2: object pointer or NULL
2614   // Z_ARG3: cache entry pointer
2615   __ call_VM(noreg,
2616              CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access),
2617              Z_ARG2, Z_ARG3);
2618   __ get_cache_and_index_at_bcp(cache, index, 1);
2619 
2620   __ bind(exit);
2621 }
2622 
2623 void TemplateTable::pop_and_check_object(Register r) {
2624   __ pop_ptr(r);
2625   __ null_check(r);  // for field access must check obj.
2626   __ verify_oop(r);
2627 }
2628 
2629 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2630   transition(vtos, vtos);
2631 
2632   const Register cache = Z_tmp_1;
2633   const Register index = Z_tmp_2;
2634   const Register obj   = Z_tmp_1;
2635   const Register off   = Z_ARG2;
2636   const Register flags = Z_ARG1;
2637   const Register bc    = Z_tmp_1;  // Uses same reg as obj, so don't mix them.
2638 
2639   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2640   jvmti_post_field_access(cache, index, is_static, false);
2641   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2642 
2643   if (!is_static) {
2644     // Obj is on the stack.
2645     pop_and_check_object(obj);
2646   }
2647 
2648   // Displacement is 0, so any store instruction will be fine on any CPU.
2649   const Address field(obj, off);
2650 
2651   Label    is_Byte, is_Bool, is_Int, is_Short, is_Char,
2652            is_Long, is_Float, is_Object, is_Double;
2653   Label    is_badState8, is_badState9, is_badStateA, is_badStateB,
2654            is_badStateC, is_badStateD, is_badStateE, is_badStateF,
2655            is_badState;
2656   Label    branchTable, atosHandler,  Done;
2657   Register br_tab       = Z_R1_scratch;
2658   bool     do_rewrite   = !is_static && (rc == may_rewrite);
2659   bool     dont_rewrite = (is_static || (rc == may_not_rewrite));
2660 
2661   assert(do_rewrite == !dont_rewrite, "Oops, code is not fit for that");
2662   assert(btos == 0, "change code, btos != 0");
2663 
2664   // Calculate branch table size. Generated code size depends on ASSERT and on bytecode rewriting.
2665 #ifdef ASSERT
2666   const unsigned int bsize = dont_rewrite ? BTB_MINSIZE*1 : BTB_MINSIZE*4;
2667 #else
2668   const unsigned int bsize = dont_rewrite ? BTB_MINSIZE*1 : BTB_MINSIZE*4;
2669 #endif
2670 
2671   // Calculate address of branch table entry and branch there.
2672   {
2673     const int bit_shift = exact_log2(bsize); // Size of each branch table entry.
2674     const int r_bitpos  = 63 - bit_shift;
2675     const int l_bitpos  = r_bitpos - ConstantPoolCacheEntry::tos_state_bits + 1;
2676     const int n_rotate  = (bit_shift-ConstantPoolCacheEntry::tos_state_shift);
2677     __ z_larl(br_tab, branchTable);
2678     __ rotate_then_insert(flags, flags, l_bitpos, r_bitpos, n_rotate, true);
2679   }
2680   __ z_bc(Assembler::bcondAlways, 0, flags, br_tab);
2681 
2682   __ align_address(bsize);
2683   BIND(branchTable);
2684 
2685   // btos
2686   BTB_BEGIN(is_Byte, bsize, "getfield_or_static:is_Byte");
2687   __ z_lb(Z_tos, field);
2688   __ push(btos);
2689   // Rewrite bytecode to be faster.
2690   if (do_rewrite) {
2691     patch_bytecode(Bytecodes::_fast_bgetfield, bc, Z_ARG5);
2692   }
2693   __ z_bru(Done);
2694   BTB_END(is_Byte, bsize, "getfield_or_static:is_Byte");
2695 
2696   // ztos
2697   BTB_BEGIN(is_Bool, bsize, "getfield_or_static:is_Bool");
2698   __ z_lb(Z_tos, field);
2699   __ push(ztos);
2700   // Rewrite bytecode to be faster.
2701   if (do_rewrite) {
2702     // Use btos rewriting, no truncating to t/f bit is needed for getfield.
2703     patch_bytecode(Bytecodes::_fast_bgetfield, bc, Z_ARG5);
2704   }
2705   __ z_bru(Done);
2706   BTB_END(is_Bool, bsize, "getfield_or_static:is_Bool");
2707 
2708   // ctos
2709   BTB_BEGIN(is_Char, bsize, "getfield_or_static:is_Char");
2710   // Load into 64 bits, works on all CPUs.
2711   __ z_llgh(Z_tos, field);
2712   __ push(ctos);
2713   // Rewrite bytecode to be faster.
2714   if (do_rewrite) {
2715     patch_bytecode(Bytecodes::_fast_cgetfield, bc, Z_ARG5);
2716   }
2717   __ z_bru(Done);
2718   BTB_END(is_Char, bsize, "getfield_or_static:is_Char");
2719 
2720   // stos
2721   BTB_BEGIN(is_Short, bsize, "getfield_or_static:is_Short");
2722   __ z_lh(Z_tos, field);
2723   __ push(stos);
2724   // Rewrite bytecode to be faster.
2725   if (do_rewrite) {
2726     patch_bytecode(Bytecodes::_fast_sgetfield, bc, Z_ARG5);
2727   }
2728   __ z_bru(Done);
2729   BTB_END(is_Short, bsize, "getfield_or_static:is_Short");
2730 
2731   // itos
2732   BTB_BEGIN(is_Int, bsize, "getfield_or_static:is_Int");
2733   __ mem2reg_opt(Z_tos, field, false);
2734   __ push(itos);
2735   // Rewrite bytecode to be faster.
2736   if (do_rewrite) {
2737     patch_bytecode(Bytecodes::_fast_igetfield, bc, Z_ARG5);
2738   }
2739   __ z_bru(Done);
2740   BTB_END(is_Int, bsize, "getfield_or_static:is_Int");
2741 
2742   // ltos
2743   BTB_BEGIN(is_Long, bsize, "getfield_or_static:is_Long");
2744   __ mem2reg_opt(Z_tos, field);
2745   __ push(ltos);
2746   // Rewrite bytecode to be faster.
2747   if (do_rewrite) {
2748     patch_bytecode(Bytecodes::_fast_lgetfield, bc, Z_ARG5);
2749   }
2750   __ z_bru(Done);
2751   BTB_END(is_Long, bsize, "getfield_or_static:is_Long");
2752 
2753   // ftos
2754   BTB_BEGIN(is_Float, bsize, "getfield_or_static:is_Float");
2755   __ mem2freg_opt(Z_ftos, field, false);
2756   __ push(ftos);
2757   // Rewrite bytecode to be faster.
2758   if (do_rewrite) {
2759     patch_bytecode(Bytecodes::_fast_fgetfield, bc, Z_ARG5);
2760   }
2761   __ z_bru(Done);
2762   BTB_END(is_Float, bsize, "getfield_or_static:is_Float");
2763 
2764   // dtos
2765   BTB_BEGIN(is_Double, bsize, "getfield_or_static:is_Double");
2766   __ mem2freg_opt(Z_ftos, field);
2767   __ push(dtos);
2768   // Rewrite bytecode to be faster.
2769   if (do_rewrite) {
2770     patch_bytecode(Bytecodes::_fast_dgetfield, bc, Z_ARG5);
2771   }
2772   __ z_bru(Done);
2773   BTB_END(is_Double, bsize, "getfield_or_static:is_Double");
2774 
2775   // atos
2776   BTB_BEGIN(is_Object, bsize, "getfield_or_static:is_Object");
2777   __ z_bru(atosHandler);
2778   BTB_END(is_Object, bsize, "getfield_or_static:is_Object");
2779 
2780   // Bad state detection comes at no extra runtime cost.
2781   BTB_BEGIN(is_badState8, bsize, "getfield_or_static:is_badState8");
2782   __ z_illtrap();
2783   __ z_bru(is_badState);
2784   BTB_END( is_badState8, bsize, "getfield_or_static:is_badState8");
2785   BTB_BEGIN(is_badState9, bsize, "getfield_or_static:is_badState9");
2786   __ z_illtrap();
2787   __ z_bru(is_badState);
2788   BTB_END( is_badState9, bsize, "getfield_or_static:is_badState9");
2789   BTB_BEGIN(is_badStateA, bsize, "getfield_or_static:is_badStateA");
2790   __ z_illtrap();
2791   __ z_bru(is_badState);
2792   BTB_END( is_badStateA, bsize, "getfield_or_static:is_badStateA");
2793   BTB_BEGIN(is_badStateB, bsize, "getfield_or_static:is_badStateB");
2794   __ z_illtrap();
2795   __ z_bru(is_badState);
2796   BTB_END( is_badStateB, bsize, "getfield_or_static:is_badStateB");
2797   BTB_BEGIN(is_badStateC, bsize, "getfield_or_static:is_badStateC");
2798   __ z_illtrap();
2799   __ z_bru(is_badState);
2800   BTB_END( is_badStateC, bsize, "getfield_or_static:is_badStateC");
2801   BTB_BEGIN(is_badStateD, bsize, "getfield_or_static:is_badStateD");
2802   __ z_illtrap();
2803   __ z_bru(is_badState);
2804   BTB_END( is_badStateD, bsize, "getfield_or_static:is_badStateD");
2805   BTB_BEGIN(is_badStateE, bsize, "getfield_or_static:is_badStateE");
2806   __ z_illtrap();
2807   __ z_bru(is_badState);
2808   BTB_END( is_badStateE, bsize, "getfield_or_static:is_badStateE");
2809   BTB_BEGIN(is_badStateF, bsize, "getfield_or_static:is_badStateF");
2810   __ z_illtrap();
2811   __ z_bru(is_badState);
2812   BTB_END( is_badStateF, bsize, "getfield_or_static:is_badStateF");
2813 
2814   __ align_address(64);
2815   BIND(is_badState);  // Do this outside branch table. Needs a lot of space.
2816   {
2817     unsigned int b_off = __ offset();
2818     if (is_static) {
2819       __ stop_static("Bad state in getstatic");
2820     } else {
2821       __ stop_static("Bad state in getfield");
2822     }
2823     unsigned int e_off = __ offset();
2824   }
2825 
2826   __ align_address(64);
2827   BIND(atosHandler);  // Oops are really complicated to handle.
2828                       // There is a lot of code generated.
2829                       // Therefore: generate the handler outside of branch table.
2830                       // There is no performance penalty. The additional branch
2831                       // to here is compensated for by the fallthru to "Done".
2832   {
2833     unsigned int b_off = __ offset();
2834     __ load_heap_oop(Z_tos, field);
2835     __ verify_oop(Z_tos);
2836     __ push(atos);
2837     if (do_rewrite) {
2838       patch_bytecode(Bytecodes::_fast_agetfield, bc, Z_ARG5);
2839     }
2840     unsigned int e_off = __ offset();
2841   }
2842 
2843   BIND(Done);
2844 }
2845 
2846 void TemplateTable::getfield(int byte_no) {
2847   BLOCK_COMMENT("getfield  {");
2848   getfield_or_static(byte_no, false);
2849   BLOCK_COMMENT("} getfield");
2850 }
2851 
2852 void TemplateTable::nofast_getfield(int byte_no) {
2853   getfield_or_static(byte_no, false, may_not_rewrite);
2854 }
2855 
2856 void TemplateTable::getstatic(int byte_no) {
2857   BLOCK_COMMENT("getstatic {");
2858   getfield_or_static(byte_no, true);
2859   BLOCK_COMMENT("} getstatic");
2860 }
2861 
2862 // The registers cache and index expected to be set before call.  The
2863 // function may destroy various registers, just not the cache and
2864 // index registers.
2865 void TemplateTable::jvmti_post_field_mod(Register cache,
2866                                          Register index, bool is_static) {
2867   transition(vtos, vtos);
2868 
2869   if (!JvmtiExport::can_post_field_modification()) {
2870     return;
2871   }
2872 
2873   BLOCK_COMMENT("jvmti_post_field_mod {");
2874 
2875   // Check to see if a field modification watch has been set before
2876   // we take the time to call into the VM.
2877   Label    L1;
2878   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2879   assert_different_registers(cache, index, Z_tos);
2880 
2881   __ load_absolute_address(Z_tos, (address)JvmtiExport::get_field_modification_count_addr());
2882   __ load_and_test_int(Z_R0, Address(Z_tos));
2883   __ z_brz(L1);
2884 
2885   // Index is returned as byte offset, do not shift!
2886   __ get_cache_and_index_at_bcp(Z_ARG3, Z_R1_scratch, 1);
2887 
2888   if (is_static) {
2889     // Life is simple. Null out the object pointer.
2890     __ clear_reg(Z_ARG2, true, false);   // Don't set CC.
2891   } else {
2892     // Life is harder. The stack holds the value on top, followed by
2893     // the object. We don't know the size of the value, though. It
2894     // could be one or two words depending on its type. As a result,
2895     // we must find the type to determine where the object is.
2896     __ mem2reg_opt(Z_ARG4,
2897                    Address(Z_ARG3, Z_R1_scratch,
2898                            in_bytes(cp_base_offset + ConstantPoolCacheEntry::flags_offset()) +
2899                            (BytesPerLong - BytesPerInt)),
2900                    false);
2901     __ z_srl(Z_ARG4, ConstantPoolCacheEntry::tos_state_shift);
2902     // Make sure we don't need to mask Z_ARG4 for tos_state after the above shift.
2903     ConstantPoolCacheEntry::verify_tos_state_shift();
2904     __ mem2reg_opt(Z_ARG2, at_tos(1));  // Initially assume a one word jvalue.
2905 
2906     NearLabel   load_dtos, cont;
2907 
2908     __ compareU32_and_branch(Z_ARG4, (intptr_t) ltos,
2909                               Assembler::bcondNotEqual, load_dtos);
2910     __ mem2reg_opt(Z_ARG2, at_tos(2)); // ltos (two word jvalue)
2911     __ z_bru(cont);
2912 
2913     __ bind(load_dtos);
2914     __ compareU32_and_branch(Z_ARG4, (intptr_t)dtos, Assembler::bcondNotEqual, cont);
2915     __ mem2reg_opt(Z_ARG2, at_tos(2)); // dtos (two word jvalue)
2916 
2917     __ bind(cont);
2918   }
2919   // cache entry pointer
2920 
2921   __ add2reg_with_index(Z_ARG3, in_bytes(cp_base_offset), Z_ARG3, Z_R1_scratch);
2922 
2923   // object(tos)
2924   __ load_address(Z_ARG4, Address(Z_esp, Interpreter::stackElementSize));
2925   // Z_ARG2: object pointer set up above (NULL if static)
2926   // Z_ARG3: cache entry pointer
2927   // Z_ARG4: jvalue object on the stack
2928   __ call_VM(noreg,
2929              CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification),
2930              Z_ARG2, Z_ARG3, Z_ARG4);
2931   __ get_cache_and_index_at_bcp(cache, index, 1);
2932 
2933   __ bind(L1);
2934   BLOCK_COMMENT("} jvmti_post_field_mod");
2935 }
2936 
2937 
2938 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2939   transition(vtos, vtos);
2940 
2941   const Register cache         = Z_tmp_1;
2942   const Register index         = Z_ARG5;
2943   const Register obj           = Z_tmp_1;
2944   const Register off           = Z_tmp_2;
2945   const Register flags         = Z_R1_scratch;
2946   const Register br_tab        = Z_ARG5;
2947   const Register bc            = Z_tmp_1;
2948   const Register oopStore_tmp1 = Z_R1_scratch;
2949   const Register oopStore_tmp2 = Z_ARG5;
2950   const Register oopStore_tmp3 = Z_R0_scratch;
2951 
2952   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2953   jvmti_post_field_mod(cache, index, is_static);
2954   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2955   // begin of life for:
2956   //   obj, off   long life range
2957   //   flags      short life range, up to branch into branch table
2958   // end of life for:
2959   //   cache, index
2960 
2961   const Address field(obj, off);
2962   Label is_Byte, is_Bool, is_Int, is_Short, is_Char,
2963         is_Long, is_Float, is_Object, is_Double;
2964   Label is_badState8, is_badState9, is_badStateA, is_badStateB,
2965         is_badStateC, is_badStateD, is_badStateE, is_badStateF,
2966         is_badState;
2967   Label branchTable, atosHandler, Done;
2968   bool  do_rewrite   = !is_static && (rc == may_rewrite);
2969   bool  dont_rewrite = (is_static || (rc == may_not_rewrite));
2970 
2971   assert(do_rewrite == !dont_rewrite, "Oops, code is not fit for that");
2972 
2973   assert(btos == 0, "change code, btos != 0");
2974 
2975 #ifdef ASSERT
2976   const unsigned int bsize = is_static ? BTB_MINSIZE*1 : BTB_MINSIZE*4;
2977 #else
2978   const unsigned int bsize = is_static ? BTB_MINSIZE*1 : BTB_MINSIZE*8;
2979 #endif
2980 
2981   // Calculate address of branch table entry and branch there.
2982   {
2983     const int bit_shift = exact_log2(bsize); // Size of each branch table entry.
2984     const int r_bitpos  = 63 - bit_shift;
2985     const int l_bitpos  = r_bitpos - ConstantPoolCacheEntry::tos_state_bits + 1;
2986     const int n_rotate  = (bit_shift-ConstantPoolCacheEntry::tos_state_shift);
2987     __ z_larl(br_tab, branchTable);
2988     __ rotate_then_insert(flags, flags, l_bitpos, r_bitpos, n_rotate, true);
2989     __ z_bc(Assembler::bcondAlways, 0, flags, br_tab);
2990   }
2991   // end of life for:
2992   //   flags, br_tab
2993 
2994   __ align_address(bsize);
2995   BIND(branchTable);
2996 
2997   // btos
2998   BTB_BEGIN(is_Byte, bsize, "putfield_or_static:is_Byte");
2999   __ pop(btos);
3000   if (!is_static) {
3001     pop_and_check_object(obj);
3002   }
3003   __ z_stc(Z_tos, field);
3004   if (do_rewrite) {
3005     patch_bytecode(Bytecodes::_fast_bputfield, bc, Z_ARG5, true, byte_no);
3006   }
3007   __ z_bru(Done);
3008   BTB_END( is_Byte, bsize, "putfield_or_static:is_Byte");
3009 
3010   // ztos
3011   BTB_BEGIN(is_Bool, bsize, "putfield_or_static:is_Bool");
3012   __ pop(ztos);
3013   if (!is_static) {
3014     pop_and_check_object(obj);
3015   }
3016   __ z_nilf(Z_tos, 0x1);
3017   __ z_stc(Z_tos, field);
3018   if (do_rewrite) {
3019     patch_bytecode(Bytecodes::_fast_zputfield, bc, Z_ARG5, true, byte_no);
3020   }
3021   __ z_bru(Done);
3022   BTB_END(is_Bool, bsize, "putfield_or_static:is_Bool");
3023 
3024   // ctos
3025   BTB_BEGIN(is_Char, bsize, "putfield_or_static:is_Char");
3026   __ pop(ctos);
3027   if (!is_static) {
3028     pop_and_check_object(obj);
3029   }
3030   __ z_sth(Z_tos, field);
3031   if (do_rewrite) {
3032     patch_bytecode(Bytecodes::_fast_cputfield, bc, Z_ARG5, true, byte_no);
3033   }
3034   __ z_bru(Done);
3035   BTB_END( is_Char, bsize, "putfield_or_static:is_Char");
3036 
3037   // stos
3038   BTB_BEGIN(is_Short, bsize, "putfield_or_static:is_Short");
3039   __ pop(stos);
3040   if (!is_static) {
3041     pop_and_check_object(obj);
3042   }
3043   __ z_sth(Z_tos, field);
3044   if (do_rewrite) {
3045     patch_bytecode(Bytecodes::_fast_sputfield, bc, Z_ARG5, true, byte_no);
3046   }
3047   __ z_bru(Done);
3048   BTB_END( is_Short, bsize, "putfield_or_static:is_Short");
3049 
3050   // itos
3051   BTB_BEGIN(is_Int, bsize, "putfield_or_static:is_Int");
3052   __ pop(itos);
3053   if (!is_static) {
3054     pop_and_check_object(obj);
3055   }
3056   __ reg2mem_opt(Z_tos, field, false);
3057   if (do_rewrite) {
3058     patch_bytecode(Bytecodes::_fast_iputfield, bc, Z_ARG5, true, byte_no);
3059   }
3060   __ z_bru(Done);
3061   BTB_END( is_Int, bsize, "putfield_or_static:is_Int");
3062 
3063   // ltos
3064   BTB_BEGIN(is_Long, bsize, "putfield_or_static:is_Long");
3065   __ pop(ltos);
3066   if (!is_static) {
3067     pop_and_check_object(obj);
3068   }
3069   __ reg2mem_opt(Z_tos, field);
3070   if (do_rewrite) {
3071     patch_bytecode(Bytecodes::_fast_lputfield, bc, Z_ARG5, true, byte_no);
3072   }
3073   __ z_bru(Done);
3074   BTB_END( is_Long, bsize, "putfield_or_static:is_Long");
3075 
3076   // ftos
3077   BTB_BEGIN(is_Float, bsize, "putfield_or_static:is_Float");
3078   __ pop(ftos);
3079   if (!is_static) {
3080     pop_and_check_object(obj);
3081   }
3082   __ freg2mem_opt(Z_ftos, field, false);
3083   if (do_rewrite) {
3084     patch_bytecode(Bytecodes::_fast_fputfield, bc, Z_ARG5, true, byte_no);
3085   }
3086   __ z_bru(Done);
3087   BTB_END( is_Float, bsize, "putfield_or_static:is_Float");
3088 
3089   // dtos
3090   BTB_BEGIN(is_Double, bsize, "putfield_or_static:is_Double");
3091   __ pop(dtos);
3092   if (!is_static) {
3093     pop_and_check_object(obj);
3094   }
3095   __ freg2mem_opt(Z_ftos, field);
3096   if (do_rewrite) {
3097     patch_bytecode(Bytecodes::_fast_dputfield, bc, Z_ARG5, true, byte_no);
3098   }
3099   __ z_bru(Done);
3100   BTB_END( is_Double, bsize, "putfield_or_static:is_Double");
3101 
3102   // atos
3103   BTB_BEGIN(is_Object, bsize, "putfield_or_static:is_Object");
3104   __ z_bru(atosHandler);
3105   BTB_END( is_Object, bsize, "putfield_or_static:is_Object");
3106 
3107   // Bad state detection comes at no extra runtime cost.
3108   BTB_BEGIN(is_badState8, bsize, "putfield_or_static:is_badState8");
3109   __ z_illtrap();
3110   __ z_bru(is_badState);
3111   BTB_END( is_badState8, bsize, "putfield_or_static:is_badState8");
3112   BTB_BEGIN(is_badState9, bsize, "putfield_or_static:is_badState9");
3113   __ z_illtrap();
3114   __ z_bru(is_badState);
3115   BTB_END( is_badState9, bsize, "putfield_or_static:is_badState9");
3116   BTB_BEGIN(is_badStateA, bsize, "putfield_or_static:is_badStateA");
3117   __ z_illtrap();
3118   __ z_bru(is_badState);
3119   BTB_END( is_badStateA, bsize, "putfield_or_static:is_badStateA");
3120   BTB_BEGIN(is_badStateB, bsize, "putfield_or_static:is_badStateB");
3121   __ z_illtrap();
3122   __ z_bru(is_badState);
3123   BTB_END( is_badStateB, bsize, "putfield_or_static:is_badStateB");
3124   BTB_BEGIN(is_badStateC, bsize, "putfield_or_static:is_badStateC");
3125   __ z_illtrap();
3126   __ z_bru(is_badState);
3127   BTB_END( is_badStateC, bsize, "putfield_or_static:is_badStateC");
3128   BTB_BEGIN(is_badStateD, bsize, "putfield_or_static:is_badStateD");
3129   __ z_illtrap();
3130   __ z_bru(is_badState);
3131   BTB_END( is_badStateD, bsize, "putfield_or_static:is_badStateD");
3132   BTB_BEGIN(is_badStateE, bsize, "putfield_or_static:is_badStateE");
3133   __ z_illtrap();
3134   __ z_bru(is_badState);
3135   BTB_END( is_badStateE, bsize, "putfield_or_static:is_badStateE");
3136   BTB_BEGIN(is_badStateF, bsize, "putfield_or_static:is_badStateF");
3137   __ z_illtrap();
3138   __ z_bru(is_badState);
3139   BTB_END( is_badStateF, bsize, "putfield_or_static:is_badStateF");
3140 
3141   __ align_address(64);
3142   BIND(is_badState);  // Do this outside branch table. Needs a lot of space.
3143   {
3144     unsigned int b_off = __ offset();
3145     if (is_static) __ stop_static("Bad state in putstatic");
3146     else            __ stop_static("Bad state in putfield");
3147     unsigned int e_off = __ offset();
3148   }
3149 
3150   __ align_address(64);
3151   BIND(atosHandler);  // Oops are really complicated to handle.
3152                       // There is a lot of code generated.
3153                       // Therefore: generate the handler outside of branch table.
3154                       // There is no performance penalty. The additional branch
3155                       // to here is compensated for by the fallthru to "Done".
3156   {
3157     unsigned int b_off = __ offset();
3158     __ pop(atos);
3159     if (!is_static) {
3160       pop_and_check_object(obj);
3161     }
3162     // Store into the field
3163     do_oop_store(_masm, obj, off, Z_tos, false,
3164                  oopStore_tmp1, oopStore_tmp2, oopStore_tmp3, _bs->kind(), false);
3165     if (do_rewrite) {
3166       patch_bytecode(Bytecodes::_fast_aputfield, bc, Z_ARG5, true, byte_no);
3167     }
3168     // __ z_bru(Done); // fallthru
3169     unsigned int e_off = __ offset();
3170   }
3171 
3172   BIND(Done);
3173 
3174   // Check for volatile store.
3175   Label notVolatile;
3176 
3177   __ testbit(Z_ARG4, ConstantPoolCacheEntry::is_volatile_shift);
3178   __ z_brz(notVolatile);
3179   __ z_fence();
3180 
3181   BIND(notVolatile);
3182 }
3183 
3184 void TemplateTable::putfield(int byte_no) {
3185   BLOCK_COMMENT("putfield  {");
3186   putfield_or_static(byte_no, false);
3187   BLOCK_COMMENT("} putfield");
3188 }
3189 
3190 void TemplateTable::nofast_putfield(int byte_no) {
3191   putfield_or_static(byte_no, false, may_not_rewrite);
3192 }
3193 
3194 void TemplateTable::putstatic(int byte_no) {
3195   BLOCK_COMMENT("putstatic {");
3196   putfield_or_static(byte_no, true);
3197   BLOCK_COMMENT("} putstatic");
3198 }
3199 
3200 // Push the tos value back to the stack.
3201 // gc will find oops there and update.
3202 void TemplateTable::jvmti_post_fast_field_mod() {
3203 
3204   if (!JvmtiExport::can_post_field_modification()) {
3205     return;
3206   }
3207 
3208   // Check to see if a field modification watch has been set before
3209   // we take the time to call into the VM.
3210   Label   exit;
3211 
3212   BLOCK_COMMENT("jvmti_post_fast_field_mod {");
3213 
3214   __ load_absolute_address(Z_R1_scratch,
3215                            (address) JvmtiExport::get_field_modification_count_addr());
3216   __ load_and_test_int(Z_R0_scratch, Address(Z_R1_scratch));
3217   __ z_brz(exit);
3218 
3219   Register obj = Z_tmp_1;
3220 
3221   __ pop_ptr(obj);                  // Copy the object pointer from tos.
3222   __ verify_oop(obj);
3223   __ push_ptr(obj);                 // Put the object pointer back on tos.
3224 
3225   // Save tos values before call_VM() clobbers them. Since we have
3226   // to do it for every data type, we use the saved values as the
3227   // jvalue object.
3228   switch (bytecode()) {          // Load values into the jvalue object.
3229     case Bytecodes::_fast_aputfield:
3230       __ push_ptr(Z_tos);
3231       break;
3232     case Bytecodes::_fast_bputfield:
3233     case Bytecodes::_fast_zputfield:
3234     case Bytecodes::_fast_sputfield:
3235     case Bytecodes::_fast_cputfield:
3236     case Bytecodes::_fast_iputfield:
3237       __ push_i(Z_tos);
3238       break;
3239     case Bytecodes::_fast_dputfield:
3240       __ push_d();
3241       break;
3242     case Bytecodes::_fast_fputfield:
3243       __ push_f();
3244       break;
3245     case Bytecodes::_fast_lputfield:
3246       __ push_l(Z_tos);
3247       break;
3248 
3249     default:
3250       ShouldNotReachHere();
3251   }
3252 
3253   // jvalue on the stack
3254   __ load_address(Z_ARG4, Address(Z_esp, Interpreter::stackElementSize));
3255   // Access constant pool cache entry.
3256   __ get_cache_entry_pointer_at_bcp(Z_ARG3, Z_tos, 1);
3257   __ verify_oop(obj);
3258 
3259   // obj   : object pointer copied above
3260   // Z_ARG3: cache entry pointer
3261   // Z_ARG4: jvalue object on the stack
3262   __ call_VM(noreg,
3263              CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification),
3264              obj, Z_ARG3, Z_ARG4);
3265 
3266   switch (bytecode()) {             // Restore tos values.
3267     case Bytecodes::_fast_aputfield:
3268       __ pop_ptr(Z_tos);
3269       break;
3270     case Bytecodes::_fast_bputfield:
3271     case Bytecodes::_fast_zputfield:
3272     case Bytecodes::_fast_sputfield:
3273     case Bytecodes::_fast_cputfield:
3274     case Bytecodes::_fast_iputfield:
3275       __ pop_i(Z_tos);
3276       break;
3277     case Bytecodes::_fast_dputfield:
3278       __ pop_d(Z_ftos);
3279       break;
3280     case Bytecodes::_fast_fputfield:
3281       __ pop_f(Z_ftos);
3282       break;
3283     case Bytecodes::_fast_lputfield:
3284       __ pop_l(Z_tos);
3285       break;
3286   }
3287 
3288   __ bind(exit);
3289   BLOCK_COMMENT("} jvmti_post_fast_field_mod");
3290 }
3291 
3292 void TemplateTable::fast_storefield(TosState state) {
3293   transition(state, vtos);
3294 
3295   ByteSize base = ConstantPoolCache::base_offset();
3296   jvmti_post_fast_field_mod();
3297 
3298   // Access constant pool cache.
3299   Register cache = Z_tmp_1;
3300   Register index = Z_tmp_2;
3301   Register flags = Z_ARG5;
3302 
3303   // Index comes in bytes, don't shift afterwards!
3304   __ get_cache_and_index_at_bcp(cache, index, 1);
3305 
3306   // Test for volatile.
3307   assert(!flags->is_volatile(), "do_oop_store could perform leaf RT call");
3308   __ z_lg(flags, Address(cache, index, base + ConstantPoolCacheEntry::flags_offset()));
3309 
3310   // Replace index with field offset from cache entry.
3311   Register field_offset = index;
3312   __ z_lg(field_offset, Address(cache, index, base + ConstantPoolCacheEntry::f2_offset()));
3313 
3314   // Get object from stack.
3315   Register   obj = cache;
3316 
3317   pop_and_check_object(obj);
3318 
3319   // field address
3320   const Address   field(obj, field_offset);
3321 
3322   // access field
3323   switch (bytecode()) {
3324     case Bytecodes::_fast_aputfield:
3325       do_oop_store(_masm, obj, field_offset, Z_tos, false,
3326                    Z_ARG2, Z_ARG3, Z_ARG4, _bs->kind(), false);
3327       break;
3328     case Bytecodes::_fast_lputfield:
3329       __ reg2mem_opt(Z_tos, field);
3330       break;
3331     case Bytecodes::_fast_iputfield:
3332       __ reg2mem_opt(Z_tos, field, false);
3333       break;
3334     case Bytecodes::_fast_zputfield:
3335       __ z_nilf(Z_tos, 0x1);
3336       // fall through to bputfield
3337     case Bytecodes::_fast_bputfield:
3338       __ z_stc(Z_tos, field);
3339       break;
3340     case Bytecodes::_fast_sputfield:
3341       // fall through
3342     case Bytecodes::_fast_cputfield:
3343       __ z_sth(Z_tos, field);
3344       break;
3345     case Bytecodes::_fast_fputfield:
3346       __ freg2mem_opt(Z_ftos, field, false);
3347       break;
3348     case Bytecodes::_fast_dputfield:
3349       __ freg2mem_opt(Z_ftos, field);
3350       break;
3351     default:
3352       ShouldNotReachHere();
3353   }
3354 
3355   //  Check for volatile store.
3356   Label notVolatile;
3357 
3358   __ testbit(flags, ConstantPoolCacheEntry::is_volatile_shift);
3359   __ z_brz(notVolatile);
3360   __ z_fence();
3361 
3362   __ bind(notVolatile);
3363 }
3364 
3365 void TemplateTable::fast_accessfield(TosState state) {
3366   transition(atos, state);
3367 
3368   Register obj = Z_tos;
3369 
3370   // Do the JVMTI work here to avoid disturbing the register state below
3371   if (JvmtiExport::can_post_field_access()) {
3372     // Check to see if a field access watch has been set before we
3373     // take the time to call into the VM.
3374     Label cont;
3375 
3376     __ load_absolute_address(Z_R1_scratch,
3377                              (address)JvmtiExport::get_field_access_count_addr());
3378     __ load_and_test_int(Z_R0_scratch, Address(Z_R1_scratch));
3379     __ z_brz(cont);
3380 
3381     // Access constant pool cache entry.
3382 
3383     __ get_cache_entry_pointer_at_bcp(Z_ARG3, Z_tmp_1, 1);
3384     __ verify_oop(obj);
3385     __ push_ptr(obj);  // Save object pointer before call_VM() clobbers it.
3386     __ z_lgr(Z_ARG2, obj);
3387 
3388     // Z_ARG2: object pointer copied above
3389     // Z_ARG3: cache entry pointer
3390     __ call_VM(noreg,
3391                CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access),
3392                Z_ARG2, Z_ARG3);
3393     __ pop_ptr(obj); // Restore object pointer.
3394 
3395     __ bind(cont);
3396   }
3397 
3398   // Access constant pool cache.
3399   Register   cache = Z_tmp_1;
3400   Register   index = Z_tmp_2;
3401 
3402   // Index comes in bytes, don't shift afterwards!
3403   __ get_cache_and_index_at_bcp(cache, index, 1);
3404   // Replace index with field offset from cache entry.
3405   __ mem2reg_opt(index,
3406                  Address(cache, index,
3407                          ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset()));
3408 
3409   __ verify_oop(obj);
3410   __ null_check(obj);
3411 
3412   Address field(obj, index);
3413 
3414   // access field
3415   switch (bytecode()) {
3416     case Bytecodes::_fast_agetfield:
3417       __ load_heap_oop(Z_tos, field);
3418       __ verify_oop(Z_tos);
3419       return;
3420     case Bytecodes::_fast_lgetfield:
3421       __ mem2reg_opt(Z_tos, field);
3422       return;
3423     case Bytecodes::_fast_igetfield:
3424       __ mem2reg_opt(Z_tos, field, false);
3425       return;
3426     case Bytecodes::_fast_bgetfield:
3427       __ z_lb(Z_tos, field);
3428       return;
3429     case Bytecodes::_fast_sgetfield:
3430       __ z_lh(Z_tos, field);
3431       return;
3432     case Bytecodes::_fast_cgetfield:
3433       __ z_llgh(Z_tos, field);   // Load into 64 bits, works on all CPUs.
3434       return;
3435     case Bytecodes::_fast_fgetfield:
3436       __ mem2freg_opt(Z_ftos, field, false);
3437       return;
3438     case Bytecodes::_fast_dgetfield:
3439       __ mem2freg_opt(Z_ftos, field);
3440       return;
3441     default:
3442       ShouldNotReachHere();
3443   }
3444 }
3445 
3446 void TemplateTable::fast_xaccess(TosState state) {
3447   transition(vtos, state);
3448 
3449   Register receiver = Z_tos;
3450   // Get receiver.
3451   __ mem2reg_opt(Z_tos, aaddress(0));
3452 
3453   // Access constant pool cache.
3454   Register cache = Z_tmp_1;
3455   Register index = Z_tmp_2;
3456 
3457   // Index comes in bytes, don't shift afterwards!
3458   __ get_cache_and_index_at_bcp(cache, index, 2);
3459   // Replace index with field offset from cache entry.
3460   __ mem2reg_opt(index,
3461                  Address(cache, index,
3462                          ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset()));
3463 
3464   // Make sure exception is reported in correct bcp range (getfield is
3465   // next instruction).
3466   __ add2reg(Z_bcp, 1);
3467   __ null_check(receiver);
3468   switch (state) {
3469     case itos:
3470       __ mem2reg_opt(Z_tos, Address(receiver, index), false);
3471       break;
3472     case atos:
3473       __ load_heap_oop(Z_tos, Address(receiver, index));
3474       __ verify_oop(Z_tos);
3475       break;
3476     case ftos:
3477       __ mem2freg_opt(Z_ftos, Address(receiver, index));
3478       break;
3479     default:
3480       ShouldNotReachHere();
3481   }
3482 
3483   // Reset bcp to original position.
3484   __ add2reg(Z_bcp, -1);
3485 }
3486 
3487 //-----------------------------------------------------------------------------
3488 // Calls
3489 
3490 void TemplateTable::prepare_invoke(int byte_no,
3491                                    Register method,  // linked method (or i-klass)
3492                                    Register index,   // itable index, MethodType, etc.
3493                                    Register recv,    // If caller wants to see it.
3494                                    Register flags) { // If caller wants to test it.
3495   // Determine flags.
3496   const Bytecodes::Code code = bytecode();
3497   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3498   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3499   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3500   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3501   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3502   const bool load_receiver       = (recv != noreg);
3503   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3504 
3505   // Setup registers & access constant pool cache.
3506   if (recv  == noreg) { recv  = Z_ARG1; }
3507   if (flags == noreg) { flags = Z_ARG2; }
3508   assert_different_registers(method, Z_R14, index, recv, flags);
3509 
3510   BLOCK_COMMENT("prepare_invoke {");
3511 
3512   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3513 
3514   // Maybe push appendix to arguments.
3515   if (is_invokedynamic || is_invokehandle) {
3516     Label L_no_push;
3517     Register resolved_reference = Z_R1_scratch;
3518     __ testbit(flags, ConstantPoolCacheEntry::has_appendix_shift);
3519     __ z_bfalse(L_no_push);
3520     // Push the appendix as a trailing parameter.
3521     // This must be done before we get the receiver,
3522     // since the parameter_size includes it.
3523     __ load_resolved_reference_at_index(resolved_reference, index);
3524     __ verify_oop(resolved_reference);
3525     __ push_ptr(resolved_reference);  // Push appendix (MethodType, CallSite, etc.).
3526     __ bind(L_no_push);
3527   }
3528 
3529   // Load receiver if needed (after appendix is pushed so parameter size is correct).
3530   if (load_receiver) {
3531     assert(!is_invokedynamic, "");
3532     // recv := int2long(flags & ConstantPoolCacheEntry::parameter_size_mask) << 3
3533     // Flags is zero-extended int2long when loaded during load_invoke_cp_cache_entry().
3534     // Only the least significant byte (psize) of flags is used.
3535     {
3536       const unsigned int logSES = Interpreter::logStackElementSize;
3537       const int bit_shift = logSES;
3538       const int r_bitpos  = 63 - bit_shift;
3539       const int l_bitpos  = r_bitpos - ConstantPoolCacheEntry::parameter_size_bits + 1;
3540       const int n_rotate  = bit_shift;
3541       assert(ConstantPoolCacheEntry::parameter_size_mask == 255, "adapt bitpositions");
3542       __ rotate_then_insert(recv, flags, l_bitpos, r_bitpos, n_rotate, true);
3543     }
3544     // Recv now contains #arguments * StackElementSize.
3545 
3546     Address recv_addr(Z_esp, recv);
3547     __ z_lg(recv, recv_addr);
3548     __ verify_oop(recv);
3549   }
3550 
3551   // Compute return type.
3552   // ret_type is used by callers (invokespecial, invokestatic) at least.
3553   Register ret_type = Z_R1_scratch;
3554   assert_different_registers(ret_type, method);
3555 
3556   const address table_addr = (address)Interpreter::invoke_return_entry_table_for(code);
3557   __ load_absolute_address(Z_R14, table_addr);
3558 
3559   {
3560     const int bit_shift = LogBytesPerWord;           // Size of each table entry.
3561     const int r_bitpos  = 63 - bit_shift;
3562     const int l_bitpos  = r_bitpos - ConstantPoolCacheEntry::tos_state_bits + 1;
3563     const int n_rotate  = bit_shift-ConstantPoolCacheEntry::tos_state_shift;
3564     __ rotate_then_insert(ret_type, flags, l_bitpos, r_bitpos, n_rotate, true);
3565     // Make sure we don't need to mask flags for tos_state after the above shift.
3566     ConstantPoolCacheEntry::verify_tos_state_shift();
3567   }
3568 
3569     __ z_lg(Z_R14, Address(Z_R14, ret_type)); // Load return address.
3570   BLOCK_COMMENT("} prepare_invoke");
3571 }
3572 
3573 
3574 void TemplateTable::invokevirtual_helper(Register index,
3575                                          Register recv,
3576                                          Register flags) {
3577   // Uses temporary registers Z_tmp_2, Z_ARG4.
3578   assert_different_registers(index, recv, Z_tmp_2, Z_ARG4);
3579 
3580   // Test for an invoke of a final method.
3581   Label notFinal;
3582 
3583   BLOCK_COMMENT("invokevirtual_helper {");
3584 
3585   __ testbit(flags, ConstantPoolCacheEntry::is_vfinal_shift);
3586   __ z_brz(notFinal);
3587 
3588   const Register method = index;  // Method must be Z_ARG3.
3589   assert(method == Z_ARG3, "method must be second argument for interpreter calling convention");
3590 
3591   // Do the call - the index is actually the method to call.
3592   // That is, f2 is a vtable index if !is_vfinal, else f2 is a method.
3593 
3594   // It's final, need a null check here!
3595   __ null_check(recv);
3596 
3597   // Profile this call.
3598   __ profile_final_call(Z_tmp_2);
3599   __ profile_arguments_type(Z_tmp_2, method, Z_ARG5, true); // Argument type profiling.
3600   __ jump_from_interpreted(method, Z_tmp_2);
3601 
3602   __ bind(notFinal);
3603 
3604   // Get receiver klass.
3605   __ null_check(recv, Z_R0_scratch, oopDesc::klass_offset_in_bytes());
3606   __ load_klass(Z_tmp_2, recv);
3607 
3608   // Profile this call.
3609   __ profile_virtual_call(Z_tmp_2, Z_ARG4, Z_ARG5);
3610 
3611   // Get target method & entry point.
3612   __ z_sllg(index, index, exact_log2(vtableEntry::size_in_bytes()));
3613   __ mem2reg_opt(method,
3614                  Address(Z_tmp_2, index,
3615                          Klass::vtable_start_offset() + in_ByteSize(vtableEntry::method_offset_in_bytes())));
3616   __ profile_arguments_type(Z_ARG4, method, Z_ARG5, true);
3617   __ jump_from_interpreted(method, Z_ARG4);
3618   BLOCK_COMMENT("} invokevirtual_helper");
3619 }
3620 
3621 void TemplateTable::invokevirtual(int byte_no) {
3622   transition(vtos, vtos);
3623 
3624   assert(byte_no == f2_byte, "use this argument");
3625   prepare_invoke(byte_no,
3626                  Z_ARG3,  // method or vtable index
3627                  noreg,   // unused itable index
3628                  Z_ARG1,  // recv
3629                  Z_ARG2); // flags
3630 
3631   // Z_ARG3 : index
3632   // Z_ARG1 : receiver
3633   // Z_ARG2 : flags
3634   invokevirtual_helper(Z_ARG3, Z_ARG1, Z_ARG2);
3635 }
3636 
3637 void TemplateTable::invokespecial(int byte_no) {
3638   transition(vtos, vtos);
3639 
3640   assert(byte_no == f1_byte, "use this argument");
3641   Register Rmethod = Z_tmp_2;
3642   prepare_invoke(byte_no, Rmethod, noreg, // Get f1 method.
3643                  Z_ARG3);   // Get receiver also for null check.
3644   __ verify_oop(Z_ARG3);
3645   __ null_check(Z_ARG3);
3646   // Do the call.
3647   __ profile_call(Z_ARG2);
3648   __ profile_arguments_type(Z_ARG2, Rmethod, Z_ARG5, false);
3649   __ jump_from_interpreted(Rmethod, Z_R1_scratch);
3650 }
3651 
3652 void TemplateTable::invokestatic(int byte_no) {
3653   transition(vtos, vtos);
3654 
3655   assert(byte_no == f1_byte, "use this argument");
3656   Register Rmethod = Z_tmp_2;
3657   prepare_invoke(byte_no, Rmethod);   // Get f1 method.
3658   // Do the call.
3659   __ profile_call(Z_ARG2);
3660   __ profile_arguments_type(Z_ARG2, Rmethod, Z_ARG5, false);
3661   __ jump_from_interpreted(Rmethod, Z_R1_scratch);
3662 }
3663 
3664 // Outdated feature, and we don't support it.
3665 void TemplateTable::fast_invokevfinal(int byte_no) {
3666   transition(vtos, vtos);
3667   assert(byte_no == f2_byte, "use this argument");
3668   __ stop("fast_invokevfinal not used on linuxs390x");
3669 }
3670 
3671 void TemplateTable::invokeinterface(int byte_no) {
3672   transition(vtos, vtos);
3673 
3674   assert(byte_no == f1_byte, "use this argument");
3675   Register klass     = Z_ARG2,
3676            method    = Z_ARG3,
3677            interface = Z_ARG4,
3678            flags     = Z_ARG5,
3679            receiver  = Z_tmp_1;
3680 
3681   BLOCK_COMMENT("invokeinterface {");
3682 
3683   prepare_invoke(byte_no, interface, method,  // Get f1 klassOop, f2 itable index.
3684                  receiver, flags);
3685 
3686   // Z_R14 (== Z_bytecode) : return entry
3687 
3688   // Special case of invokeinterface called for virtual method of
3689   // java.lang.Object. See cpCacheOop.cpp for details.
3690   // This code isn't produced by javac, but could be produced by
3691   // another compliant java compiler.
3692   NearLabel notMethod, no_such_interface, no_such_method;
3693   __ testbit(flags, ConstantPoolCacheEntry::is_forced_virtual_shift);
3694   __ z_brz(notMethod);
3695   invokevirtual_helper(method, receiver, flags);
3696   __ bind(notMethod);
3697 
3698   // Get receiver klass into klass - also a null check.
3699   __ restore_locals();
3700   __ load_klass(klass, receiver);
3701 
3702   __ lookup_interface_method(klass, interface, noreg, noreg, /*temp*/Z_ARG1,
3703                              no_such_interface, /*return_method=*/false);
3704 
3705   // Profile this call.
3706   __ profile_virtual_call(klass, Z_ARG1/*mdp*/, flags/*scratch*/);
3707 
3708   // Find entry point to call.
3709 
3710   // Get declaring interface class from method
3711   __ z_lg(interface, Address(method, Method::const_offset()));
3712   __ z_lg(interface, Address(interface, ConstMethod::constants_offset()));
3713   __ z_lg(interface, Address(interface, ConstantPool::pool_holder_offset_in_bytes()));
3714 
3715   // Get itable index from method
3716   Register index   = receiver,
3717            method2 = flags;
3718   __ z_lgf(index, Address(method, Method::itable_index_offset()));
3719   __ z_aghi(index, -Method::itable_index_max);
3720   __ z_lcgr(index, index);
3721 
3722   __ lookup_interface_method(klass, interface, index, method2, Z_tmp_2,
3723                              no_such_interface);
3724 
3725   // Check for abstract method error.
3726   // Note: This should be done more efficiently via a throw_abstract_method_error
3727   // interpreter entry point and a conditional jump to it in case of a null
3728   // method.
3729   __ compareU64_and_branch(method2, (intptr_t) 0,
3730                             Assembler::bcondZero, no_such_method);
3731 
3732   __ profile_arguments_type(Z_tmp_1, method2, Z_tmp_2, true);
3733 
3734   // Do the call.
3735   __ jump_from_interpreted(method2, Z_tmp_2);
3736   __ should_not_reach_here();
3737 
3738   // exception handling code follows...
3739   // Note: Must restore interpreter registers to canonical
3740   // state for exception handling to work correctly!
3741 
3742   __ bind(no_such_method);
3743 
3744   // Throw exception.
3745   __ restore_bcp();      // Bcp must be correct for exception handler   (was destroyed).
3746   __ restore_locals();   // Make sure locals pointer is correct as well (was destroyed).
3747   // Pass arguments for generating a verbose error message.
3748   __ z_lgr(Z_tmp_1, method); // Prevent register clash.
3749   __ call_VM(noreg,
3750              CAST_FROM_FN_PTR(address,
3751                               InterpreterRuntime::throw_AbstractMethodErrorVerbose),
3752                               klass, Z_tmp_1);
3753   // The call_VM checks for exception, so we should never return here.
3754   __ should_not_reach_here();
3755 
3756   __ bind(no_such_interface);
3757 
3758   // Throw exception.
3759   __ restore_bcp();      // Bcp must be correct for exception handler   (was destroyed).
3760   __ restore_locals();   // Make sure locals pointer is correct as well (was destroyed).
3761   // Pass arguments for generating a verbose error message.
3762   __ call_VM(noreg,
3763              CAST_FROM_FN_PTR(address,
3764                               InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose),
3765                               klass, interface);
3766   // The call_VM checks for exception, so we should never return here.
3767   __ should_not_reach_here();
3768 
3769   BLOCK_COMMENT("} invokeinterface");
3770   return;
3771 }
3772 
3773 void TemplateTable::invokehandle(int byte_no) {
3774   transition(vtos, vtos);
3775 
3776   const Register method = Z_tmp_2;
3777   const Register recv   = Z_ARG5;
3778   const Register mtype  = Z_tmp_1;
3779   prepare_invoke(byte_no,
3780                  method, mtype,   // Get f2 method, f1 MethodType.
3781                  recv);
3782   __ verify_method_ptr(method);
3783   __ verify_oop(recv);
3784   __ null_check(recv);
3785 
3786   // Note: Mtype is already pushed (if necessary) by prepare_invoke.
3787 
3788   // FIXME: profile the LambdaForm also.
3789   __ profile_final_call(Z_ARG2);
3790   __ profile_arguments_type(Z_ARG3, method, Z_ARG5, true);
3791 
3792   __ jump_from_interpreted(method, Z_ARG3);
3793 }
3794 
3795 void TemplateTable::invokedynamic(int byte_no) {
3796   transition(vtos, vtos);
3797 
3798   const Register Rmethod   = Z_tmp_2;
3799   const Register Rcallsite = Z_tmp_1;
3800 
3801   prepare_invoke(byte_no, Rmethod, Rcallsite);
3802 
3803   // Rmethod: CallSite object (from f1)
3804   // Rcallsite: MH.linkToCallSite method (from f2)
3805 
3806   // Note: Callsite is already pushed by prepare_invoke.
3807 
3808   // TODO: should make a type profile for any invokedynamic that takes a ref argument.
3809   // Profile this call.
3810   __ profile_call(Z_ARG2);
3811   __ profile_arguments_type(Z_ARG2, Rmethod, Z_ARG5, false);
3812   __ jump_from_interpreted(Rmethod, Z_ARG2);
3813 }
3814 
3815 //-----------------------------------------------------------------------------
3816 // Allocation
3817 
3818 // Original comment on "allow_shared_alloc":
3819 // Always go the slow path.
3820 //  + Eliminated optimization within the template-based interpreter:
3821 //    If an allocation is done within the interpreter without using
3822 //    tlabs, the interpreter tries to do the allocation directly
3823 //    on the heap.
3824 //  + That means the profiling hooks are not considered and allocations
3825 //    get lost for the profiling framework.
3826 //  + However, we do not think that this optimization is really needed,
3827 //    so we always go now the slow path through the VM in this case --
3828 //    spec jbb2005 shows no measurable performance degradation.
3829 void TemplateTable::_new() {
3830   transition(vtos, atos);
3831   address prev_instr_address = NULL;
3832   Register tags  = Z_tmp_1;
3833   Register RallocatedObject   = Z_tos;
3834   Register cpool = Z_ARG2;
3835   Register tmp = Z_ARG3; // RobjectFields==tmp and Rsize==offset must be a register pair.
3836   Register offset = Z_ARG4;
3837   Label slow_case;
3838   Label done;
3839   Label initialize_header;
3840   Label allocate_shared;
3841 
3842   BLOCK_COMMENT("TemplateTable::_new {");
3843   __ get_2_byte_integer_at_bcp(offset/*dest*/, 1, InterpreterMacroAssembler::Unsigned);
3844   __ get_cpool_and_tags(cpool, tags);
3845   // Make sure the class we're about to instantiate has been resolved.
3846   // This is done before loading InstanceKlass to be consistent with the order
3847   // how Constant Pool is updated (see ConstantPool::klass_at_put).
3848   const int tags_offset = Array<u1>::base_offset_in_bytes();
3849   __ load_address(tmp, Address(tags, offset, tags_offset));
3850   __ z_cli(0, tmp, JVM_CONSTANT_Class);
3851   __ z_brne(slow_case);
3852 
3853   __ z_sllg(offset, offset, LogBytesPerWord); // Convert to to offset.
3854   // Get InstanceKlass.
3855   Register iklass = cpool;
3856   __ load_resolved_klass_at_offset(cpool, offset, iklass);
3857 
3858   // Make sure klass is initialized & doesn't have finalizer.
3859   // Make sure klass is fully initialized.
3860   const int state_offset = in_bytes(InstanceKlass::init_state_offset());
3861   if (Immediate::is_uimm12(state_offset)) {
3862     __ z_cli(state_offset, iklass, InstanceKlass::fully_initialized);
3863   } else {
3864     __ z_cliy(state_offset, iklass, InstanceKlass::fully_initialized);
3865   }
3866   __ z_brne(slow_case);
3867 
3868   // Get instance_size in InstanceKlass (scaled to a count of bytes).
3869   Register Rsize = offset;
3870   const int mask = 1 << Klass::_lh_instance_slow_path_bit;
3871   __ z_llgf(Rsize, Address(iklass, Klass::layout_helper_offset()));
3872   __ z_tmll(Rsize, mask);
3873   __ z_btrue(slow_case);
3874 
3875   // Allocate the instance
3876   // 1) Try to allocate in the TLAB.
3877   // 2) If the above fails (or is not applicable), go to a slow case
3878   // (creates a new TLAB, etc.).
3879   // Note: compared to other architectures, s390's implementation always goes
3880   // to the slow path if TLAB is used and fails.
3881   if (UseTLAB) {
3882     Register RoldTopValue = RallocatedObject;
3883     Register RnewTopValue = tmp;
3884     __ z_lg(RoldTopValue, Address(Z_thread, JavaThread::tlab_top_offset()));
3885     __ load_address(RnewTopValue, Address(RoldTopValue, Rsize));
3886     __ z_cg(RnewTopValue, Address(Z_thread, JavaThread::tlab_end_offset()));
3887     __ z_brh(slow_case);
3888     __ z_stg(RnewTopValue, Address(Z_thread, JavaThread::tlab_top_offset()));
3889 
3890     Register RobjectFields = tmp;
3891     Register Rzero = Z_R1_scratch;
3892     __ clear_reg(Rzero, true /*whole reg*/, false); // Load 0L into Rzero. Don't set CC.
3893 
3894     if (!ZeroTLAB) {
3895       // The object is initialized before the header. If the object size is
3896       // zero, go directly to the header initialization.
3897       __ z_aghi(Rsize, (int)-sizeof(oopDesc)); // Subtract header size, set CC.
3898       __ z_bre(initialize_header);             // Jump if size of fields is zero.
3899 
3900       // Initialize object fields.
3901       // See documentation for MVCLE instruction!!!
3902       assert(RobjectFields->encoding() % 2 == 0, "RobjectFields must be an even register");
3903       assert(Rsize->encoding() == (RobjectFields->encoding()+1),
3904              "RobjectFields and Rsize must be a register pair");
3905       assert(Rzero->encoding() % 2 == 1, "Rzero must be an odd register");
3906 
3907       // Set Rzero to 0 and use it as src length, then mvcle will copy nothing
3908       // and fill the object with the padding value 0.
3909       __ add2reg(RobjectFields, sizeof(oopDesc), RallocatedObject);
3910       __ move_long_ext(RobjectFields, as_Register(Rzero->encoding() - 1), 0);
3911     }
3912 
3913     // Initialize object header only.
3914     __ bind(initialize_header);
3915     if (UseBiasedLocking) {
3916       Register prototype = RobjectFields;
3917       __ z_lg(prototype, Address(iklass, Klass::prototype_header_offset()));
3918       __ z_stg(prototype, Address(RallocatedObject, oopDesc::mark_offset_in_bytes()));
3919     } else {
3920       __ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()),
3921                      (long)markOopDesc::prototype());
3922     }
3923 
3924     __ store_klass_gap(Rzero, RallocatedObject);  // Zero klass gap for compressed oops.
3925     __ store_klass(iklass, RallocatedObject);     // Store klass last.
3926 
3927     {
3928       SkipIfEqual skip(_masm, &DTraceAllocProbes, false, Z_ARG5 /*scratch*/);
3929       // Trigger dtrace event for fastpath.
3930       __ push(atos); // Save the return value.
3931       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), RallocatedObject);
3932       __ pop(atos); // Restore the return value.
3933     }
3934     __ z_bru(done);
3935   }
3936 
3937   // slow case
3938   __ bind(slow_case);
3939   __ get_constant_pool(Z_ARG2);
3940   __ get_2_byte_integer_at_bcp(Z_ARG3/*dest*/, 1, InterpreterMacroAssembler::Unsigned);
3941   call_VM(Z_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), Z_ARG2, Z_ARG3);
3942   __ verify_oop(Z_tos);
3943 
3944   // continue
3945   __ bind(done);
3946 
3947   BLOCK_COMMENT("} TemplateTable::_new");
3948 }
3949 
3950 void TemplateTable::newarray() {
3951   transition(itos, atos);
3952 
3953   // Call runtime.
3954   __ z_llgc(Z_ARG2, at_bcp(1));   // type
3955   __ z_lgfr(Z_ARG3, Z_tos);       // size
3956   call_VM(Z_RET,
3957           CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3958           Z_ARG2, Z_ARG3);
3959 }
3960 
3961 void TemplateTable::anewarray() {
3962   transition(itos, atos);
3963   __ get_2_byte_integer_at_bcp(Z_ARG3, 1, InterpreterMacroAssembler::Unsigned);
3964   __ get_constant_pool(Z_ARG2);
3965   __ z_lgfr(Z_ARG4, Z_tos);
3966   call_VM(Z_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3967           Z_ARG2, Z_ARG3, Z_ARG4);
3968 }
3969 
3970 void TemplateTable::arraylength() {
3971   transition(atos, itos);
3972 
3973   int offset = arrayOopDesc::length_offset_in_bytes();
3974 
3975   __ null_check(Z_tos, Z_R0_scratch, offset);
3976   __ mem2reg_opt(Z_tos, Address(Z_tos, offset), false);
3977 }
3978 
3979 void TemplateTable::checkcast() {
3980   transition(atos, atos);
3981 
3982   NearLabel done, is_null, ok_is_subtype, quicked, resolved;
3983 
3984   BLOCK_COMMENT("checkcast {");
3985   // If object is NULL, we are almost done.
3986   __ compareU64_and_branch(Z_tos, (intptr_t) 0, Assembler::bcondZero, is_null);
3987 
3988   // Get cpool & tags index.
3989   Register cpool = Z_tmp_1;
3990   Register tags = Z_tmp_2;
3991   Register index = Z_ARG5;
3992 
3993   __ get_cpool_and_tags(cpool, tags);
3994   __ get_2_byte_integer_at_bcp(index, 1, InterpreterMacroAssembler::Unsigned);
3995   // See if bytecode has already been quicked.
3996   // Note: For CLI, we would have to add the index to the tags pointer first,
3997   // thus load and compare in a "classic" manner.
3998   __ z_llgc(Z_R0_scratch,
3999             Address(tags, index, Array<u1>::base_offset_in_bytes()));
4000   __ compareU64_and_branch(Z_R0_scratch, JVM_CONSTANT_Class,
4001                            Assembler::bcondEqual, quicked);
4002 
4003   __ push(atos); // Save receiver for result, and for GC.
4004   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4005   __ get_vm_result_2(Z_tos);
4006 
4007   Register   receiver = Z_ARG4;
4008   Register   klass = Z_tos;
4009   Register   subklass = Z_ARG5;
4010 
4011   __ pop_ptr(receiver); // restore receiver
4012   __ z_bru(resolved);
4013 
4014   // Get superklass in klass and subklass in subklass.
4015   __ bind(quicked);
4016 
4017   __ z_lgr(Z_ARG4, Z_tos);  // Save receiver.
4018   __ z_sllg(index, index, LogBytesPerWord);  // index2bytes for addressing
4019   __ load_resolved_klass_at_offset(cpool, index, klass);
4020 
4021   __ bind(resolved);
4022 
4023   __ load_klass(subklass, receiver);
4024 
4025   // Generate subtype check. Object in receiver.
4026   // Superklass in klass. Subklass in subklass.
4027   __ gen_subtype_check(subklass, klass, Z_ARG3, Z_tmp_1, ok_is_subtype);
4028 
4029   // Come here on failure.
4030   __ push_ptr(receiver);
4031   // Object is at TOS, target klass oop expected in rax by convention.
4032   __ z_brul((address) Interpreter::_throw_ClassCastException_entry);
4033 
4034   // Come here on success.
4035   __ bind(ok_is_subtype);
4036 
4037   __ z_lgr(Z_tos, receiver); // Restore object.
4038 
4039   // Collect counts on whether this test sees NULLs a lot or not.
4040   if (ProfileInterpreter) {
4041     __ z_bru(done);
4042     __ bind(is_null);
4043     __ profile_null_seen(Z_tmp_1);
4044   } else {
4045     __ bind(is_null);   // Same as 'done'.
4046   }
4047 
4048   __ bind(done);
4049   BLOCK_COMMENT("} checkcast");
4050 }
4051 
4052 void TemplateTable::instanceof() {
4053   transition(atos, itos);
4054 
4055   NearLabel done, is_null, ok_is_subtype, quicked, resolved;
4056 
4057   BLOCK_COMMENT("instanceof {");
4058   // If object is NULL, we are almost done.
4059   __ compareU64_and_branch(Z_tos, (intptr_t) 0, Assembler::bcondZero, is_null);
4060 
4061   // Get cpool & tags index.
4062   Register cpool = Z_tmp_1;
4063   Register tags = Z_tmp_2;
4064   Register index = Z_ARG5;
4065 
4066   __ get_cpool_and_tags(cpool, tags);
4067   __ get_2_byte_integer_at_bcp(index, 1, InterpreterMacroAssembler::Unsigned);
4068   // See if bytecode has already been quicked.
4069   // Note: For CLI, we would have to add the index to the tags pointer first,
4070   // thus load and compare in a "classic" manner.
4071   __ z_llgc(Z_R0_scratch,
4072             Address(tags, index, Array<u1>::base_offset_in_bytes()));
4073   __ compareU64_and_branch(Z_R0_scratch, JVM_CONSTANT_Class, Assembler::bcondEqual, quicked);
4074 
4075   __ push(atos); // Save receiver for result, and for GC.
4076   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4077   __ get_vm_result_2(Z_tos);
4078 
4079   Register receiver = Z_tmp_2;
4080   Register klass = Z_tos;
4081   Register subklass = Z_tmp_2;
4082 
4083   __ pop_ptr(receiver); // Restore receiver.
4084   __ verify_oop(receiver);
4085   __ load_klass(subklass, subklass);
4086   __ z_bru(resolved);
4087 
4088   // Get superklass in klass and subklass in subklass.
4089   __ bind(quicked);
4090 
4091   __ load_klass(subklass, Z_tos);
4092   __ z_sllg(index, index, LogBytesPerWord);  // index2bytes for addressing
4093   __ load_resolved_klass_at_offset(cpool, index, klass);
4094 
4095   __ bind(resolved);
4096 
4097   // Generate subtype check.
4098   // Superklass in klass. Subklass in subklass.
4099   __ gen_subtype_check(subklass, klass, Z_ARG4, Z_ARG5, ok_is_subtype);
4100 
4101   // Come here on failure.
4102   __ clear_reg(Z_tos, true, false);
4103   __ z_bru(done);
4104 
4105   // Come here on success.
4106   __ bind(ok_is_subtype);
4107   __ load_const_optimized(Z_tos, 1);
4108 
4109   // Collect counts on whether this test sees NULLs a lot or not.
4110   if (ProfileInterpreter) {
4111     __ z_bru(done);
4112     __ bind(is_null);
4113     __ profile_null_seen(Z_tmp_1);
4114   } else {
4115     __ bind(is_null);   // same as 'done'
4116   }
4117 
4118   __ bind(done);
4119   // tos = 0: obj == NULL or  obj is not an instanceof the specified klass
4120   // tos = 1: obj != NULL and obj is     an instanceof the specified klass
4121   BLOCK_COMMENT("} instanceof");
4122 }
4123 
4124 //-----------------------------------------------------------------------------
4125 // Breakpoints
4126 void TemplateTable::_breakpoint() {
4127 
4128   // Note: We get here even if we are single stepping.
4129   // Jbug insists on setting breakpoints at every bytecode
4130   // even if we are in single step mode.
4131 
4132   transition(vtos, vtos);
4133 
4134   // Get the unpatched byte code.
4135   __ get_method(Z_ARG2);
4136   __ call_VM(noreg,
4137              CAST_FROM_FN_PTR(address, InterpreterRuntime::get_original_bytecode_at),
4138              Z_ARG2, Z_bcp);
4139   // Save the result to a register that is preserved over C-function calls.
4140   __ z_lgr(Z_tmp_1, Z_RET);
4141 
4142   // Post the breakpoint event.
4143   __ get_method(Z_ARG2);
4144   __ call_VM(noreg,
4145              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
4146              Z_ARG2, Z_bcp);
4147 
4148   // Must restore the bytecode, because call_VM destroys Z_bytecode.
4149   __ z_lgr(Z_bytecode, Z_tmp_1);
4150 
4151   // Complete the execution of original bytecode.
4152   __ dispatch_only_normal(vtos);
4153 }
4154 
4155 
4156 // Exceptions
4157 
4158 void TemplateTable::athrow() {
4159   transition(atos, vtos);
4160   __ null_check(Z_tos);
4161   __ load_absolute_address(Z_ARG2, Interpreter::throw_exception_entry());
4162   __ z_br(Z_ARG2);
4163 }
4164 
4165 // Synchronization
4166 //
4167 // Note: monitorenter & exit are symmetric routines; which is reflected
4168 //       in the assembly code structure as well
4169 //
4170 // Stack layout:
4171 //
4172 //               callers_sp        <- Z_SP (callers_sp == Z_fp (own fp))
4173 //               return_pc
4174 //               [rest of ABI_160]
4175 //              /slot o:   free
4176 //             / ...       free
4177 //       oper. | slot n+1: free    <- Z_esp points to first free slot
4178 //       stack | slot n:   val                      caches IJAVA_STATE.esp
4179 //             | ...
4180 //              \slot 0:   val
4181 //              /slot m            <- IJAVA_STATE.monitors = monitor block top
4182 //             | ...
4183 //     monitors| slot 2
4184 //             | slot 1
4185 //              \slot 0
4186 //              /slot l            <- monitor block bot
4187 // ijava_state | ...
4188 //             | slot 2
4189 //              \slot 0
4190 //                                 <- Z_fp
4191 void TemplateTable::monitorenter() {
4192   transition(atos, vtos);
4193 
4194   BLOCK_COMMENT("monitorenter {");
4195 
4196   // Check for NULL object.
4197   __ null_check(Z_tos);
4198   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4199   NearLabel allocated;
4200   // Initialize entry pointer.
4201   const Register Rfree_slot = Z_tmp_1;
4202   __ clear_reg(Rfree_slot, true, false); // Points to free slot or NULL. Don't set CC.
4203 
4204   // Find a free slot in the monitor block from top to bot (result in Rfree_slot).
4205   {
4206     const Register Rcurr_monitor = Z_ARG2;
4207     const Register Rbot = Z_ARG3; // Points to word under bottom of monitor block.
4208     const Register Rlocked_obj = Z_ARG4;
4209     NearLabel loop, exit, not_free;
4210     // Starting with top-most entry.
4211     __ get_monitors(Rcurr_monitor); // Rcur_monitor = IJAVA_STATE.monitors
4212     __ add2reg(Rbot, -frame::z_ijava_state_size, Z_fp);
4213 
4214 #ifdef ASSERT
4215     address reentry = NULL;
4216     { NearLabel ok;
4217       __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondNotHigh, ok);
4218       reentry = __ stop_chain_static(reentry, "IJAVA_STATE.monitors points below monitor block bottom");
4219       __ bind(ok);
4220     }
4221     { NearLabel ok;
4222       __ compareU64_and_branch(Rcurr_monitor, Z_esp, Assembler::bcondHigh, ok);
4223       reentry = __ stop_chain_static(reentry, "IJAVA_STATE.monitors above Z_esp");
4224       __ bind(ok);
4225     }
4226 #endif
4227 
4228     // Check if bottom reached, i.e. if there is at least one monitor.
4229     __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondEqual, exit);
4230 
4231     __ bind(loop);
4232     // Check if current entry is used.
4233     __ load_and_test_long(Rlocked_obj, Address(Rcurr_monitor, BasicObjectLock::obj_offset_in_bytes()));
4234     __ z_brne(not_free);
4235     // If not used then remember entry in Rfree_slot.
4236     __ z_lgr(Rfree_slot, Rcurr_monitor);
4237     __ bind(not_free);
4238     // Exit if current entry is for same object; this guarantees, that new monitor
4239     // used for recursive lock is above the older one.
4240     __ compareU64_and_branch(Rlocked_obj, Z_tos, Assembler::bcondEqual, exit);
4241     // otherwise advance to next entry
4242     __ add2reg(Rcurr_monitor, entry_size);
4243     // Check if bottom reached, if not at bottom then check this entry.
4244     __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondNotEqual, loop);
4245     __ bind(exit);
4246   }
4247 
4248   // Rfree_slot != NULL -> found one
4249   __ compareU64_and_branch(Rfree_slot, (intptr_t)0L, Assembler::bcondNotEqual, allocated);
4250 
4251   // Allocate one if there's no free slot.
4252   __ add_monitor_to_stack(false, Z_ARG3, Z_ARG4, Z_ARG5);
4253   __ get_monitors(Rfree_slot);
4254 
4255   // Rfree_slot: points to monitor entry.
4256   __ bind(allocated);
4257 
4258   // Increment bcp to point to the next bytecode, so exception
4259   // handling for async. exceptions work correctly.
4260   // The object has already been poped from the stack, so the
4261   // expression stack looks correct.
4262   __ add2reg(Z_bcp, 1, Z_bcp);
4263 
4264   // Store object.
4265   __ z_stg(Z_tos, BasicObjectLock::obj_offset_in_bytes(), Rfree_slot);
4266   __ lock_object(Rfree_slot, Z_tos);
4267 
4268   // Check to make sure this monitor doesn't cause stack overflow after locking.
4269   __ save_bcp();  // in case of exception
4270   __ generate_stack_overflow_check(0);
4271 
4272   // The bcp has already been incremented. Just need to dispatch to
4273   // next instruction.
4274   __ dispatch_next(vtos);
4275 
4276   BLOCK_COMMENT("} monitorenter");
4277 }
4278 
4279 
4280 void TemplateTable::monitorexit() {
4281   transition(atos, vtos);
4282 
4283   BLOCK_COMMENT("monitorexit {");
4284 
4285   // Check for NULL object.
4286   __ null_check(Z_tos);
4287 
4288   NearLabel found, not_found;
4289   const Register Rcurr_monitor = Z_ARG2;
4290 
4291   // Find matching slot.
4292   {
4293     const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4294     NearLabel entry, loop;
4295 
4296     const Register Rbot = Z_ARG3; // Points to word under bottom of monitor block.
4297     const Register Rlocked_obj = Z_ARG4;
4298     // Starting with top-most entry.
4299     __ get_monitors(Rcurr_monitor); // Rcur_monitor = IJAVA_STATE.monitors
4300     __ add2reg(Rbot, -frame::z_ijava_state_size, Z_fp);
4301 
4302 #ifdef ASSERT
4303     address reentry = NULL;
4304     { NearLabel ok;
4305       __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondNotHigh, ok);
4306       reentry = __ stop_chain_static(reentry, "IJAVA_STATE.monitors points below monitor block bottom");
4307       __ bind(ok);
4308     }
4309     { NearLabel ok;
4310       __ compareU64_and_branch(Rcurr_monitor, Z_esp, Assembler::bcondHigh, ok);
4311       reentry = __ stop_chain_static(reentry, "IJAVA_STATE.monitors above Z_esp");
4312       __ bind(ok);
4313     }
4314 #endif
4315 
4316     // Check if bottom reached, i.e. if there is at least one monitor.
4317     __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondEqual, not_found);
4318 
4319     __ bind(loop);
4320     // Check if current entry is for same object.
4321     __ z_lg(Rlocked_obj, Address(Rcurr_monitor, BasicObjectLock::obj_offset_in_bytes()));
4322     // If same object then stop searching.
4323     __ compareU64_and_branch(Rlocked_obj, Z_tos, Assembler::bcondEqual, found);
4324     // Otherwise advance to next entry.
4325     __ add2reg(Rcurr_monitor, entry_size);
4326     // Check if bottom reached, if not at bottom then check this entry.
4327     __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondNotEqual, loop);
4328   }
4329 
4330   __ bind(not_found);
4331   // Error handling. Unlocking was not block-structured.
4332   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4333                    InterpreterRuntime::throw_illegal_monitor_state_exception));
4334   __ should_not_reach_here();
4335 
4336   __ bind(found);
4337   __ push_ptr(Z_tos); // Make sure object is on stack (contract with oopMaps).
4338   __ unlock_object(Rcurr_monitor, Z_tos);
4339   __ pop_ptr(Z_tos); // Discard object.
4340   BLOCK_COMMENT("} monitorexit");
4341 }
4342 
4343 // Wide instructions
4344 void TemplateTable::wide() {
4345   transition(vtos, vtos);
4346 
4347   __ z_llgc(Z_R1_scratch, at_bcp(1));
4348   __ z_sllg(Z_R1_scratch, Z_R1_scratch, LogBytesPerWord);
4349   __ load_absolute_address(Z_tmp_1, (address) Interpreter::_wentry_point);
4350   __ mem2reg_opt(Z_tmp_1, Address(Z_tmp_1, Z_R1_scratch));
4351   __ z_br(Z_tmp_1);
4352   // Note: the bcp increment step is part of the individual wide
4353   // bytecode implementations.
4354 }
4355 
4356 // Multi arrays
4357 void TemplateTable::multianewarray() {
4358   transition(vtos, atos);
4359 
4360   __ z_llgc(Z_tmp_1, at_bcp(3)); // Get number of dimensions.
4361   // Slot count to byte offset.
4362   __ z_sllg(Z_tmp_1, Z_tmp_1, Interpreter::logStackElementSize);
4363   // Z_esp points past last_dim, so set to Z_ARG2 to first_dim address.
4364   __ load_address(Z_ARG2, Address(Z_esp, Z_tmp_1));
4365   call_VM(Z_RET,
4366           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
4367           Z_ARG2);
4368   // Pop dimensions from expression stack.
4369   __ z_agr(Z_esp, Z_tmp_1);
4370 }