1 /*
   2  * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2016, 2017 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "interpreter/interpreterRuntime.hpp"
  30 #include "interpreter/interp_masm.hpp"
  31 #include "interpreter/templateTable.hpp"
  32 #include "memory/universe.inline.hpp"
  33 #include "oops/objArrayKlass.hpp"
  34 #include "oops/oop.inline.hpp"
  35 #include "prims/methodHandles.hpp"
  36 #include "runtime/sharedRuntime.hpp"
  37 #include "runtime/stubRoutines.hpp"
  38 #include "runtime/synchronizer.hpp"
  39 
  40 #ifdef PRODUCT
  41 #define __ _masm->
  42 #define BLOCK_COMMENT(str)
  43 #define BIND(label)        __ bind(label);
  44 #else
  45 #define __ (PRODUCT_ONLY(false&&)Verbose ? (_masm->block_comment(FILE_AND_LINE),_masm):_masm)->
  46 #define BLOCK_COMMENT(str) __ block_comment(str)
  47 #define BIND(label)        __ bind(label); BLOCK_COMMENT(#label ":")
  48 #endif
  49 
  50 // The assumed minimum size of a BranchTableBlock.
  51 // The actual size of each block heavily depends on the CPU capabilities and,
  52 // of course, on the logic implemented in each block.
  53 #ifdef ASSERT
  54   #define BTB_MINSIZE 256
  55 #else
  56   #define BTB_MINSIZE  64
  57 #endif
  58 
  59 #ifdef ASSERT
  60 // Macro to open a BranchTableBlock (a piece of code that is branched to by a calculated branch).
  61 #define BTB_BEGIN(lbl, alignment, name)                                        \
  62   __ align_address(alignment);                                                 \
  63   __ bind(lbl);                                                                \
  64   { unsigned int b_off = __ offset();                                          \
  65     uintptr_t   b_addr = (uintptr_t)__ pc();                                   \
  66     __ z_larl(Z_R0, (int64_t)0);     /* Check current address alignment. */    \
  67     __ z_slgr(Z_R0, br_tab);         /* Current Address must be equal    */    \
  68     __ z_slgr(Z_R0, flags);          /* to calculated branch target.     */    \
  69     __ z_brc(Assembler::bcondLogZero, 3); /* skip trap if ok. */               \
  70     __ z_illtrap(0x55);                                                        \
  71     guarantee(b_addr%alignment == 0, "bad alignment at begin of block" name);
  72 
  73 // Macro to close a BranchTableBlock (a piece of code that is branched to by a calculated branch).
  74 #define BTB_END(lbl, alignment, name)                                          \
  75     uintptr_t   e_addr = (uintptr_t)__ pc();                                   \
  76     unsigned int e_off = __ offset();                                          \
  77     unsigned int len   = e_off-b_off;                                          \
  78     if (len > alignment) {                                                     \
  79       tty->print_cr("%4d of %4d @ " INTPTR_FORMAT ": Block len for %s",        \
  80                     len, alignment, e_addr-len, name);                         \
  81       guarantee(len <= alignment, "block too large");                          \
  82     }                                                                          \
  83     guarantee(len == e_addr-b_addr, "block len mismatch");                     \
  84   }
  85 #else
  86 // Macro to open a BranchTableBlock (a piece of code that is branched to by a calculated branch).
  87 #define BTB_BEGIN(lbl, alignment, name)                                        \
  88   __ align_address(alignment);                                                 \
  89   __ bind(lbl);                                                                \
  90   { unsigned int b_off = __ offset();                                          \
  91     uintptr_t   b_addr = (uintptr_t)__ pc();                                   \
  92     guarantee(b_addr%alignment == 0, "bad alignment at begin of block" name);
  93 
  94 // Macro to close a BranchTableBlock (a piece of code that is branched to by a calculated branch).
  95 #define BTB_END(lbl, alignment, name)                                          \
  96     uintptr_t   e_addr = (uintptr_t)__ pc();                                   \
  97     unsigned int e_off = __ offset();                                          \
  98     unsigned int len   = e_off-b_off;                                          \
  99     if (len > alignment) {                                                     \
 100       tty->print_cr("%4d of %4d @ " INTPTR_FORMAT ": Block len for %s",        \
 101                     len, alignment, e_addr-len, name);                         \
 102       guarantee(len <= alignment, "block too large");                          \
 103     }                                                                          \
 104     guarantee(len == e_addr-b_addr, "block len mismatch");                     \
 105   }
 106 #endif // ASSERT
 107 
 108 // Platform-dependent initialization.
 109 
 110 void TemplateTable::pd_initialize() {
 111   // No specific initialization.
 112 }
 113 
 114 // Address computation: local variables
 115 
 116 static inline Address iaddress(int n) {
 117   return Address(Z_locals, Interpreter::local_offset_in_bytes(n));
 118 }
 119 
 120 static inline Address laddress(int n) {
 121   return iaddress(n + 1);
 122 }
 123 
 124 static inline Address faddress(int n) {
 125   return iaddress(n);
 126 }
 127 
 128 static inline Address daddress(int n) {
 129   return laddress(n);
 130 }
 131 
 132 static inline Address aaddress(int n) {
 133   return iaddress(n);
 134 }
 135 
 136 // Pass NULL, if no shift instruction should be emitted.
 137 static inline Address iaddress(InterpreterMacroAssembler *masm, Register r) {
 138   if (masm) {
 139     masm->z_sllg(r, r, LogBytesPerWord);  // index2bytes
 140   }
 141   return Address(Z_locals, r, Interpreter::local_offset_in_bytes(0));
 142 }
 143 
 144 // Pass NULL, if no shift instruction should be emitted.
 145 static inline Address laddress(InterpreterMacroAssembler *masm, Register r) {
 146   if (masm) {
 147     masm->z_sllg(r, r, LogBytesPerWord);  // index2bytes
 148   }
 149   return Address(Z_locals, r, Interpreter::local_offset_in_bytes(1) );
 150 }
 151 
 152 static inline Address faddress(InterpreterMacroAssembler *masm, Register r) {
 153   return iaddress(masm, r);
 154 }
 155 
 156 static inline Address daddress(InterpreterMacroAssembler *masm, Register r) {
 157   return laddress(masm, r);
 158 }
 159 
 160 static inline Address aaddress(InterpreterMacroAssembler *masm, Register r) {
 161   return iaddress(masm, r);
 162 }
 163 
 164 // At top of Java expression stack which may be different than esp(). It
 165 // isn't for category 1 objects.
 166 static inline Address at_tos(int slot = 0) {
 167   return Address(Z_esp, Interpreter::expr_offset_in_bytes(slot));
 168 }
 169 
 170 // Condition conversion
 171 static Assembler::branch_condition j_not(TemplateTable::Condition cc) {
 172   switch (cc) {
 173     case TemplateTable::equal :
 174       return Assembler::bcondNotEqual;
 175     case TemplateTable::not_equal :
 176       return Assembler::bcondEqual;
 177     case TemplateTable::less :
 178       return Assembler::bcondNotLow;
 179     case TemplateTable::less_equal :
 180       return Assembler::bcondHigh;
 181     case TemplateTable::greater :
 182       return Assembler::bcondNotHigh;
 183     case TemplateTable::greater_equal:
 184       return Assembler::bcondLow;
 185   }
 186   ShouldNotReachHere();
 187   return Assembler::bcondZero;
 188 }
 189 
 190 // Do an oop store like *(base + offset) = val
 191 // offset can be a register or a constant.
 192 static void do_oop_store(InterpreterMacroAssembler* _masm,
 193                          Register base,
 194                          RegisterOrConstant offset,
 195                          Register val,
 196                          bool val_is_null, // == false does not guarantee that val really is not equal NULL.
 197                          Register tmp1,    // If tmp3 is volatile, either tmp1 or tmp2 must be
 198                          Register tmp2,    // non-volatile to hold a copy of pre_val across runtime calls.
 199                          Register tmp3,    // Ideally, this tmp register is non-volatile, as it is used to
 200                                            // hold pre_val (must survive runtime calls).
 201                          BarrierSet::Name barrier,
 202                          bool precise) {
 203   BLOCK_COMMENT("do_oop_store {");
 204   assert(val != noreg, "val must always be valid, even if it is zero");
 205   assert_different_registers(tmp1, tmp2, tmp3, val, base, offset.register_or_noreg());
 206   __ verify_oop(val);
 207   switch (barrier) {
 208 #if INCLUDE_ALL_GCS
 209     case BarrierSet::G1SATBCTLogging:
 210       {
 211 #ifdef ASSERT
 212         if (val_is_null) { // Check if the flag setting reflects reality.
 213           Label OK;
 214           __ z_ltgr(val, val);
 215           __ z_bre(OK);
 216           __ z_illtrap(0x11);
 217           __ bind(OK);
 218         }
 219 #endif
 220         Register pre_val = tmp3;
 221         // Load and record the previous value.
 222         __ g1_write_barrier_pre(base, offset, pre_val, val,
 223                                 tmp1, tmp2,
 224                                 false);  // Needs to hold pre_val in non_volatile register?
 225 
 226         if (val_is_null) {
 227           __ store_heap_oop_null(val, offset, base);
 228         } else {
 229           Label Done;
 230           // val_is_null == false does not guarantee that val really is not equal NULL.
 231           // Checking for this case dynamically has some cost, but also some benefit (in GC).
 232           // It's hard to say if cost or benefit is greater.
 233           { Label OK;
 234             __ z_ltgr(val, val);
 235             __ z_brne(OK);
 236             __ store_heap_oop_null(val, offset, base);
 237             __ z_bru(Done);
 238             __ bind(OK);
 239           }
 240           // G1 barrier needs uncompressed oop for region cross check.
 241           // Store_heap_oop compresses the oop in the argument register.
 242           Register val_work = val;
 243           if (UseCompressedOops) {
 244             val_work = tmp3;
 245             __ z_lgr(val_work, val);
 246           }
 247           __ store_heap_oop_not_null(val_work, offset, base);
 248 
 249           // We need precise card marks for oop array stores.
 250           // Otherwise, cardmarking the object which contains the oop is sufficient.
 251           if (precise && !(offset.is_constant() && offset.as_constant() == 0)) {
 252             __ add2reg_with_index(base,
 253                                   offset.constant_or_zero(),
 254                                   offset.register_or_noreg(),
 255                                   base);
 256           }
 257           __ g1_write_barrier_post(base /* store_adr */, val, tmp1, tmp2, tmp3);
 258           __ bind(Done);
 259         }
 260       }
 261       break;
 262 #endif // INCLUDE_ALL_GCS
 263     case BarrierSet::CardTableModRef:
 264     {
 265       if (val_is_null) {
 266         __ store_heap_oop_null(val, offset, base);
 267       } else {
 268         __ store_heap_oop(val, offset, base);
 269         // Flatten object address if needed.
 270         if (precise && ((offset.register_or_noreg() != noreg) || (offset.constant_or_zero() != 0))) {
 271           __ load_address(base, Address(base, offset.register_or_noreg(), offset.constant_or_zero()));
 272         }
 273         __ card_write_barrier_post(base, tmp1);
 274       }
 275     }
 276     break;
 277   case BarrierSet::ModRef:
 278     // fall through
 279   default:
 280     ShouldNotReachHere();
 281 
 282   }
 283   BLOCK_COMMENT("} do_oop_store");
 284 }
 285 
 286 Address TemplateTable::at_bcp(int offset) {
 287   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 288   return Address(Z_bcp, offset);
 289 }
 290 
 291 void TemplateTable::patch_bytecode(Bytecodes::Code bc,
 292                                    Register        bc_reg,
 293                                    Register        temp_reg,
 294                                    bool            load_bc_into_bc_reg, // = true
 295                                    int             byte_no) {
 296   if (!RewriteBytecodes) { return; }
 297 
 298   NearLabel L_patch_done;
 299   BLOCK_COMMENT("patch_bytecode {");
 300 
 301   switch (bc) {
 302     case Bytecodes::_fast_aputfield:
 303     case Bytecodes::_fast_bputfield:
 304     case Bytecodes::_fast_zputfield:
 305     case Bytecodes::_fast_cputfield:
 306     case Bytecodes::_fast_dputfield:
 307     case Bytecodes::_fast_fputfield:
 308     case Bytecodes::_fast_iputfield:
 309     case Bytecodes::_fast_lputfield:
 310     case Bytecodes::_fast_sputfield:
 311       {
 312         // We skip bytecode quickening for putfield instructions when
 313         // the put_code written to the constant pool cache is zero.
 314         // This is required so that every execution of this instruction
 315         // calls out to InterpreterRuntime::resolve_get_put to do
 316         // additional, required work.
 317         assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 318         assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 319         __ get_cache_and_index_and_bytecode_at_bcp(Z_R1_scratch, bc_reg,
 320                                                    temp_reg, byte_no, 1);
 321         __ load_const_optimized(bc_reg, bc);
 322         __ compareU32_and_branch(temp_reg, (intptr_t)0,
 323                                  Assembler::bcondZero, L_patch_done);
 324       }
 325       break;
 326     default:
 327       assert(byte_no == -1, "sanity");
 328       // The pair bytecodes have already done the load.
 329       if (load_bc_into_bc_reg) {
 330         __ load_const_optimized(bc_reg, bc);
 331       }
 332       break;
 333   }
 334 
 335   if (JvmtiExport::can_post_breakpoint()) {
 336 
 337     Label   L_fast_patch;
 338 
 339     // If a breakpoint is present we can't rewrite the stream directly.
 340     __ z_cli(at_bcp(0), Bytecodes::_breakpoint);
 341     __ z_brne(L_fast_patch);
 342     __ get_method(temp_reg);
 343     // Let breakpoint table handling rewrite to quicker bytecode.
 344     __ call_VM_static(noreg,
 345                       CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at),
 346                       temp_reg, Z_R13, bc_reg);
 347     __ z_bru(L_patch_done);
 348 
 349     __ bind(L_fast_patch);
 350   }
 351 
 352 #ifdef ASSERT
 353   NearLabel   L_okay;
 354 
 355   // We load into 64 bits, since this works on any CPU.
 356   __ z_llgc(temp_reg, at_bcp(0));
 357   __ compareU32_and_branch(temp_reg, Bytecodes::java_code(bc),
 358                             Assembler::bcondEqual, L_okay        );
 359   __ compareU32_and_branch(temp_reg, bc_reg, Assembler::bcondEqual, L_okay);
 360   __ stop_static("patching the wrong bytecode");
 361   __ bind(L_okay);
 362 #endif
 363 
 364   // Patch bytecode.
 365   __ z_stc(bc_reg, at_bcp(0));
 366 
 367   __ bind(L_patch_done);
 368   BLOCK_COMMENT("} patch_bytecode");
 369 }
 370 
 371 // Individual instructions
 372 
 373 void TemplateTable::nop() {
 374   transition(vtos, vtos);
 375 }
 376 
 377 void TemplateTable::shouldnotreachhere() {
 378   transition(vtos, vtos);
 379   __ stop("shouldnotreachhere bytecode");
 380 }
 381 
 382 void TemplateTable::aconst_null() {
 383   transition(vtos, atos);
 384   __ clear_reg(Z_tos, true, false);
 385 }
 386 
 387 void TemplateTable::iconst(int value) {
 388   transition(vtos, itos);
 389   // Zero extension of the iconst makes zero extension at runtime obsolete.
 390   __ load_const_optimized(Z_tos, ((unsigned long)(unsigned int)value));
 391 }
 392 
 393 void TemplateTable::lconst(int value) {
 394   transition(vtos, ltos);
 395   __ load_const_optimized(Z_tos, value);
 396 }
 397 
 398 // No pc-relative load/store for floats.
 399 void TemplateTable::fconst(int value) {
 400   transition(vtos, ftos);
 401   static float   one = 1.0f, two = 2.0f;
 402 
 403   switch (value) {
 404     case 0:
 405       __ z_lzer(Z_ftos);
 406       return;
 407     case 1:
 408       __ load_absolute_address(Z_R1_scratch, (address) &one);
 409       __ mem2freg_opt(Z_ftos, Address(Z_R1_scratch), false);
 410       return;
 411     case 2:
 412       __ load_absolute_address(Z_R1_scratch, (address) &two);
 413       __ mem2freg_opt(Z_ftos, Address(Z_R1_scratch), false);
 414       return;
 415     default:
 416       ShouldNotReachHere();
 417       return;
 418   }
 419 }
 420 
 421 void TemplateTable::dconst(int value) {
 422   transition(vtos, dtos);
 423   static double one = 1.0;
 424 
 425   switch (value) {
 426     case 0:
 427       __ z_lzdr(Z_ftos);
 428       return;
 429     case 1:
 430       __ load_absolute_address(Z_R1_scratch, (address) &one);
 431       __ mem2freg_opt(Z_ftos, Address(Z_R1_scratch));
 432       return;
 433     default:
 434       ShouldNotReachHere();
 435       return;
 436   }
 437 }
 438 
 439 void TemplateTable::bipush() {
 440   transition(vtos, itos);
 441   __ z_lb(Z_tos, at_bcp(1));
 442 }
 443 
 444 void TemplateTable::sipush() {
 445   transition(vtos, itos);
 446   __ get_2_byte_integer_at_bcp(Z_tos, 1, InterpreterMacroAssembler::Signed);
 447 }
 448 
 449 
 450 void TemplateTable::ldc(bool wide) {
 451   transition(vtos, vtos);
 452   Label call_ldc, notFloat, notClass, notInt, Done;
 453   const Register RcpIndex = Z_tmp_1;
 454   const Register Rtags = Z_ARG2;
 455 
 456   if (wide) {
 457     __ get_2_byte_integer_at_bcp(RcpIndex, 1, InterpreterMacroAssembler::Unsigned);
 458   } else {
 459     __ z_llgc(RcpIndex, at_bcp(1));
 460   }
 461 
 462   __ get_cpool_and_tags(Z_tmp_2, Rtags);
 463 
 464   const int      base_offset = ConstantPool::header_size() * wordSize;
 465   const int      tags_offset = Array<u1>::base_offset_in_bytes();
 466   const Register Raddr_type = Rtags;
 467 
 468   // Get address of type.
 469   __ add2reg_with_index(Raddr_type, tags_offset, RcpIndex, Rtags);
 470 
 471   __ z_cli(0, Raddr_type, JVM_CONSTANT_UnresolvedClass);
 472   __ z_bre(call_ldc);    // Unresolved class - get the resolved class.
 473 
 474   __ z_cli(0, Raddr_type, JVM_CONSTANT_UnresolvedClassInError);
 475   __ z_bre(call_ldc);    // Unresolved class in error state - call into runtime
 476                          // to throw the error from the first resolution attempt.
 477 
 478   __ z_cli(0, Raddr_type, JVM_CONSTANT_Class);
 479   __ z_brne(notClass);   // Resolved class - need to call vm to get java
 480                          // mirror of the class.
 481 
 482   // We deal with a class. Call vm to do the appropriate.
 483   __ bind(call_ldc);
 484   __ load_const_optimized(Z_ARG2, wide);
 485   call_VM(Z_RET, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), Z_ARG2);
 486   __ push_ptr(Z_RET);
 487   __ z_bru(Done);
 488 
 489   // Not a class.
 490   __ bind(notClass);
 491   Register RcpOffset = RcpIndex;
 492   __ z_sllg(RcpOffset, RcpIndex, LogBytesPerWord); // Convert index to offset.
 493   __ z_cli(0, Raddr_type, JVM_CONSTANT_Float);
 494   __ z_brne(notFloat);
 495 
 496   // ftos
 497   __ mem2freg_opt(Z_ftos, Address(Z_tmp_2, RcpOffset, base_offset), false);
 498   __ push_f();
 499   __ z_bru(Done);
 500 
 501   __ bind(notFloat);
 502   __ z_cli(0, Raddr_type, JVM_CONSTANT_Integer);
 503   __ z_brne(notInt);
 504 
 505   // itos
 506   __ mem2reg_opt(Z_tos, Address(Z_tmp_2, RcpOffset, base_offset), false);
 507   __ push_i(Z_tos);
 508   __ z_bru(Done);
 509 
 510   // assume the tag is for condy; if not, the VM runtime will tell us
 511   __ bind(notInt);
 512   condy_helper(Done);
 513 
 514   __ bind(Done);
 515 }
 516 
 517 // Fast path for caching oop constants.
 518 // %%% We should use this to handle Class and String constants also.
 519 // %%% It will simplify the ldc/primitive path considerably.
 520 void TemplateTable::fast_aldc(bool wide) {
 521   transition(vtos, atos);
 522 
 523   const Register index = Z_tmp_2;
 524   int            index_size = wide ? sizeof(u2) : sizeof(u1);
 525   Label          L_do_resolve, L_resolved;
 526 
 527   // We are resolved if the resolved reference cache entry contains a
 528   // non-null object (CallSite, etc.).
 529   __ get_cache_index_at_bcp(index, 1, index_size);  // Load index.
 530   __ load_resolved_reference_at_index(Z_tos, index);
 531   __ z_ltgr(Z_tos, Z_tos);
 532   __ z_bre(L_do_resolve);
 533 
 534   // Convert null sentinel to NULL.
 535   __ load_const_optimized(Z_R1_scratch, (intptr_t)Universe::the_null_sentinel_addr());
 536   __ z_cg(Z_tos, Address(Z_R1_scratch));
 537   __ z_brne(L_resolved);
 538   __ clear_reg(Z_tos);
 539   __ z_bru(L_resolved);
 540 
 541   __ bind(L_do_resolve);
 542   // First time invocation - must resolve first.
 543   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 544   __ load_const_optimized(Z_ARG1, (int)bytecode());
 545   __ call_VM(Z_tos, entry, Z_ARG1);
 546 
 547   __ bind(L_resolved);
 548   __ verify_oop(Z_tos);
 549 }
 550 
 551 void TemplateTable::ldc2_w() {
 552   transition(vtos, vtos);
 553   Label notDouble, notLong, Done;
 554 
 555   // Z_tmp_1 = index of cp entry
 556   __ get_2_byte_integer_at_bcp(Z_tmp_1, 1, InterpreterMacroAssembler::Unsigned);
 557 
 558   __ get_cpool_and_tags(Z_tmp_2, Z_tos);
 559 
 560   const int base_offset = ConstantPool::header_size() * wordSize;
 561   const int tags_offset = Array<u1>::base_offset_in_bytes();
 562 
 563   // Get address of type.
 564   __ add2reg_with_index(Z_tos, tags_offset, Z_tos, Z_tmp_1);
 565 
 566   // Index needed in both branches, so calculate here.
 567   __ z_sllg(Z_tmp_1, Z_tmp_1, LogBytesPerWord);  // index2bytes
 568 
 569   // Check type.
 570   __ z_cli(0, Z_tos, JVM_CONSTANT_Double);
 571   __ z_brne(notDouble);
 572   // dtos
 573   __ mem2freg_opt(Z_ftos, Address(Z_tmp_2, Z_tmp_1, base_offset));
 574   __ push_d();
 575   __ z_bru(Done);
 576 
 577   __ bind(notDouble);
 578   __ z_cli(0, Z_tos, JVM_CONSTANT_Long);
 579   __ z_brne(notLong);
 580   // ltos
 581   __ mem2reg_opt(Z_tos, Address(Z_tmp_2, Z_tmp_1, base_offset));
 582   __ push_l();
 583   __ z_bru(Done);
 584 
 585   __ bind(notLong);
 586   condy_helper(Done);
 587 
 588   __ bind(Done);
 589 }
 590 
 591 void TemplateTable::condy_helper(Label& Done) {
 592   const Register obj   = Z_tmp_1;
 593   const Register off   = Z_tmp_2;
 594   const Register flags = Z_ARG1;
 595   const Register rarg  = Z_ARG2;
 596   __ load_const_optimized(rarg, (int)bytecode());
 597   call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc), rarg);
 598   __ get_vm_result_2(flags);
 599 
 600   // VMr = obj = base address to find primitive value to push
 601   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 602   assert(ConstantPoolCacheEntry::field_index_mask == 0xffff, "or use other instructions");
 603   __ z_llghr(off, flags);
 604   const Address field(obj, off);
 605 
 606   // What sort of thing are we loading?
 607   __ z_srl(flags, ConstantPoolCacheEntry::tos_state_shift);
 608   // Make sure we don't need to mask flags for tos_state after the above shift.
 609   ConstantPoolCacheEntry::verify_tos_state_shift();
 610 
 611   switch (bytecode()) {
 612   case Bytecodes::_ldc:
 613   case Bytecodes::_ldc_w:
 614     {
 615       // tos in (itos, ftos, stos, btos, ctos, ztos)
 616       Label notInt, notFloat, notShort, notByte, notChar, notBool;
 617       __ z_cghi(flags, itos);
 618       __ z_brne(notInt);
 619       // itos
 620       __ z_l(Z_tos, field);
 621       __ push(itos);
 622       __ z_bru(Done);
 623 
 624       __ bind(notInt);
 625       __ z_cghi(flags, ftos);
 626       __ z_brne(notFloat);
 627       // ftos
 628       __ z_le(Z_ftos, field);
 629       __ push(ftos);
 630       __ z_bru(Done);
 631 
 632       __ bind(notFloat);
 633       __ z_cghi(flags, stos);
 634       __ z_brne(notShort);
 635       // stos
 636       __ z_lh(Z_tos, field);
 637       __ push(stos);
 638       __ z_bru(Done);
 639 
 640       __ bind(notShort);
 641       __ z_cghi(flags, btos);
 642       __ z_brne(notByte);
 643       // btos
 644       __ z_lb(Z_tos, field);
 645       __ push(btos);
 646       __ z_bru(Done);
 647 
 648       __ bind(notByte);
 649       __ z_cghi(flags, ctos);
 650       __ z_brne(notChar);
 651       // ctos
 652       __ z_llh(Z_tos, field);
 653       __ push(ctos);
 654       __ z_bru(Done);
 655 
 656       __ bind(notChar);
 657       __ z_cghi(flags, ztos);
 658       __ z_brne(notBool);
 659       // ztos
 660       __ z_lb(Z_tos, field);
 661       __ push(ztos);
 662       __ z_bru(Done);
 663 
 664       __ bind(notBool);
 665       break;
 666     }
 667 
 668   case Bytecodes::_ldc2_w:
 669     {
 670       Label notLong, notDouble;
 671       __ z_cghi(flags, ltos);
 672       __ z_brne(notLong);
 673       // ltos
 674       __ z_lg(Z_tos, field);
 675       __ push(ltos);
 676       __ z_bru(Done);
 677 
 678       __ bind(notLong);
 679       __ z_cghi(flags, dtos);
 680       __ z_brne(notDouble);
 681       // dtos
 682       __ z_ld(Z_ftos, field);
 683       __ push(dtos);
 684       __ z_bru(Done);
 685 
 686       __ bind(notDouble);
 687       break;
 688     }
 689 
 690   default:
 691     ShouldNotReachHere();
 692   }
 693 
 694   __ stop("bad ldc/condy");
 695 }
 696 
 697 void TemplateTable::locals_index(Register reg, int offset) {
 698   __ z_llgc(reg, at_bcp(offset));
 699   __ z_lcgr(reg);
 700 }
 701 
 702 void TemplateTable::iload() {
 703   iload_internal();
 704 }
 705 
 706 void TemplateTable::nofast_iload() {
 707   iload_internal(may_not_rewrite);
 708 }
 709 
 710 void TemplateTable::iload_internal(RewriteControl rc) {
 711   transition(vtos, itos);
 712 
 713   if (RewriteFrequentPairs && rc == may_rewrite) {
 714     NearLabel rewrite, done;
 715     const Register bc = Z_ARG4;
 716 
 717     assert(Z_R1_scratch != bc, "register damaged");
 718 
 719     // Get next byte.
 720     __ z_llgc(Z_R1_scratch, at_bcp(Bytecodes::length_for (Bytecodes::_iload)));
 721 
 722     // If _iload, wait to rewrite to iload2. We only want to rewrite the
 723     // last two iloads in a pair. Comparing against fast_iload means that
 724     // the next bytecode is neither an iload or a caload, and therefore
 725     // an iload pair.
 726     __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_iload,
 727                              Assembler::bcondEqual, done);
 728 
 729     __ load_const_optimized(bc, Bytecodes::_fast_iload2);
 730     __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_fast_iload,
 731                              Assembler::bcondEqual, rewrite);
 732 
 733     // If _caload, rewrite to fast_icaload.
 734     __ load_const_optimized(bc, Bytecodes::_fast_icaload);
 735     __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_caload,
 736                              Assembler::bcondEqual, rewrite);
 737 
 738     // Rewrite so iload doesn't check again.
 739     __ load_const_optimized(bc, Bytecodes::_fast_iload);
 740 
 741     // rewrite
 742     // bc: fast bytecode
 743     __ bind(rewrite);
 744     patch_bytecode(Bytecodes::_iload, bc, Z_R1_scratch, false);
 745 
 746     __ bind(done);
 747 
 748   }
 749 
 750   // Get the local value into tos.
 751   locals_index(Z_R1_scratch);
 752   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
 753 }
 754 
 755 void TemplateTable::fast_iload2() {
 756   transition(vtos, itos);
 757 
 758   locals_index(Z_R1_scratch);
 759   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
 760   __ push_i(Z_tos);
 761   locals_index(Z_R1_scratch, 3);
 762   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
 763 }
 764 
 765 void TemplateTable::fast_iload() {
 766   transition(vtos, itos);
 767 
 768   locals_index(Z_R1_scratch);
 769   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
 770 }
 771 
 772 void TemplateTable::lload() {
 773   transition(vtos, ltos);
 774 
 775   locals_index(Z_R1_scratch);
 776   __ mem2reg_opt(Z_tos, laddress(_masm, Z_R1_scratch));
 777 }
 778 
 779 void TemplateTable::fload() {
 780   transition(vtos, ftos);
 781 
 782   locals_index(Z_R1_scratch);
 783   __ mem2freg_opt(Z_ftos, faddress(_masm, Z_R1_scratch), false);
 784 }
 785 
 786 void TemplateTable::dload() {
 787   transition(vtos, dtos);
 788 
 789   locals_index(Z_R1_scratch);
 790   __ mem2freg_opt(Z_ftos, daddress(_masm, Z_R1_scratch));
 791 }
 792 
 793 void TemplateTable::aload() {
 794   transition(vtos, atos);
 795 
 796   locals_index(Z_R1_scratch);
 797   __ mem2reg_opt(Z_tos, aaddress(_masm, Z_R1_scratch));
 798 }
 799 
 800 void TemplateTable::locals_index_wide(Register reg) {
 801   __ get_2_byte_integer_at_bcp(reg, 2, InterpreterMacroAssembler::Unsigned);
 802   __ z_lcgr(reg);
 803 }
 804 
 805 void TemplateTable::wide_iload() {
 806   transition(vtos, itos);
 807 
 808   locals_index_wide(Z_tmp_1);
 809   __ mem2reg_opt(Z_tos, iaddress(_masm, Z_tmp_1), false);
 810 }
 811 
 812 void TemplateTable::wide_lload() {
 813   transition(vtos, ltos);
 814 
 815   locals_index_wide(Z_tmp_1);
 816   __ mem2reg_opt(Z_tos, laddress(_masm, Z_tmp_1));
 817 }
 818 
 819 void TemplateTable::wide_fload() {
 820   transition(vtos, ftos);
 821 
 822   locals_index_wide(Z_tmp_1);
 823   __ mem2freg_opt(Z_ftos, faddress(_masm, Z_tmp_1), false);
 824 }
 825 
 826 void TemplateTable::wide_dload() {
 827   transition(vtos, dtos);
 828 
 829   locals_index_wide(Z_tmp_1);
 830   __ mem2freg_opt(Z_ftos, daddress(_masm, Z_tmp_1));
 831 }
 832 
 833 void TemplateTable::wide_aload() {
 834   transition(vtos, atos);
 835 
 836   locals_index_wide(Z_tmp_1);
 837   __ mem2reg_opt(Z_tos, aaddress(_masm, Z_tmp_1));
 838 }
 839 
 840 void TemplateTable::index_check(Register array, Register index, unsigned int shift) {
 841   assert_different_registers(Z_R1_scratch, array, index);
 842 
 843   // Check array.
 844   __ null_check(array, Z_R0_scratch, arrayOopDesc::length_offset_in_bytes());
 845 
 846   // Sign extend index for use by indexed load.
 847   __ z_lgfr(index, index);
 848 
 849   // Check index.
 850   Label index_ok;
 851   __ z_cl(index, Address(array, arrayOopDesc::length_offset_in_bytes()));
 852   __ z_brl(index_ok);
 853   __ lgr_if_needed(Z_ARG3, index); // See generate_ArrayIndexOutOfBounds_handler().
 854   // Give back the array to create more detailed exceptions.
 855   __ lgr_if_needed(Z_ARG2, array); // See generate_ArrayIndexOutOfBounds_handler().
 856   __ load_absolute_address(Z_R1_scratch,
 857                            Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 858   __ z_bcr(Assembler::bcondAlways, Z_R1_scratch);
 859   __ bind(index_ok);
 860 
 861   if (shift > 0)
 862     __ z_sllg(index, index, shift);
 863 }
 864 
 865 void TemplateTable::iaload() {
 866   transition(itos, itos);
 867 
 868   __ pop_ptr(Z_tmp_1);  // array
 869   // Index is in Z_tos.
 870   Register index = Z_tos;
 871   index_check(Z_tmp_1, index, LogBytesPerInt); // Kills Z_ARG3.
 872   // Load the value.
 873   __ mem2reg_opt(Z_tos,
 874                  Address(Z_tmp_1, index, arrayOopDesc::base_offset_in_bytes(T_INT)),
 875                  false);
 876 }
 877 
 878 void TemplateTable::laload() {
 879   transition(itos, ltos);
 880 
 881   __ pop_ptr(Z_tmp_2);
 882   // Z_tos   : index
 883   // Z_tmp_2 : array
 884   Register index = Z_tos;
 885   index_check(Z_tmp_2, index, LogBytesPerLong);
 886   __ mem2reg_opt(Z_tos,
 887                  Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_LONG)));
 888 }
 889 
 890 void TemplateTable::faload() {
 891   transition(itos, ftos);
 892 
 893   __ pop_ptr(Z_tmp_2);
 894   // Z_tos   : index
 895   // Z_tmp_2 : array
 896   Register index = Z_tos;
 897   index_check(Z_tmp_2, index, LogBytesPerInt);
 898   __ mem2freg_opt(Z_ftos,
 899                   Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_FLOAT)),
 900                   false);
 901 }
 902 
 903 void TemplateTable::daload() {
 904   transition(itos, dtos);
 905 
 906   __ pop_ptr(Z_tmp_2);
 907   // Z_tos   : index
 908   // Z_tmp_2 : array
 909   Register index = Z_tos;
 910   index_check(Z_tmp_2, index, LogBytesPerLong);
 911   __ mem2freg_opt(Z_ftos,
 912                   Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
 913 }
 914 
 915 void TemplateTable::aaload() {
 916   transition(itos, atos);
 917 
 918   unsigned const int shift = LogBytesPerHeapOop;
 919   __ pop_ptr(Z_tmp_1);  // array
 920   // Index is in Z_tos.
 921   Register index = Z_tos;
 922   index_check(Z_tmp_1, index, shift);
 923   // Now load array element.
 924   __ load_heap_oop(Z_tos,
 925                    Address(Z_tmp_1, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 926   __ verify_oop(Z_tos);
 927 }
 928 
 929 void TemplateTable::baload() {
 930   transition(itos, itos);
 931 
 932   __ pop_ptr(Z_tmp_1);
 933   // Z_tos   : index
 934   // Z_tmp_1 : array
 935   Register index = Z_tos;
 936   index_check(Z_tmp_1, index, 0);
 937   __ z_lb(Z_tos,
 938           Address(Z_tmp_1, index, arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 939 }
 940 
 941 void TemplateTable::caload() {
 942   transition(itos, itos);
 943 
 944   __ pop_ptr(Z_tmp_2);
 945   // Z_tos   : index
 946   // Z_tmp_2 : array
 947   Register index = Z_tos;
 948   index_check(Z_tmp_2, index, LogBytesPerShort);
 949   // Load into 64 bits, works on all CPUs.
 950   __ z_llgh(Z_tos,
 951             Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 952 }
 953 
 954 // Iload followed by caload frequent pair.
 955 void TemplateTable::fast_icaload() {
 956   transition(vtos, itos);
 957 
 958   // Load index out of locals.
 959   locals_index(Z_R1_scratch);
 960   __ mem2reg_opt(Z_ARG3, iaddress(_masm, Z_R1_scratch), false);
 961   // Z_ARG3  : index
 962   // Z_tmp_2 : array
 963   __ pop_ptr(Z_tmp_2);
 964   index_check(Z_tmp_2, Z_ARG3, LogBytesPerShort);
 965   // Load into 64 bits, works on all CPUs.
 966   __ z_llgh(Z_tos,
 967             Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 968 }
 969 
 970 void TemplateTable::saload() {
 971   transition(itos, itos);
 972 
 973   __ pop_ptr(Z_tmp_2);
 974   // Z_tos   : index
 975   // Z_tmp_2 : array
 976   Register index = Z_tos;
 977   index_check(Z_tmp_2, index, LogBytesPerShort);
 978   __ z_lh(Z_tos,
 979           Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 980 }
 981 
 982 void TemplateTable::iload(int n) {
 983   transition(vtos, itos);
 984   __ z_ly(Z_tos, iaddress(n));
 985 }
 986 
 987 void TemplateTable::lload(int n) {
 988   transition(vtos, ltos);
 989   __ z_lg(Z_tos, laddress(n));
 990 }
 991 
 992 void TemplateTable::fload(int n) {
 993   transition(vtos, ftos);
 994   __ mem2freg_opt(Z_ftos, faddress(n), false);
 995 }
 996 
 997 void TemplateTable::dload(int n) {
 998   transition(vtos, dtos);
 999   __ mem2freg_opt(Z_ftos, daddress(n));
1000 }
1001 
1002 void TemplateTable::aload(int n) {
1003   transition(vtos, atos);
1004   __ mem2reg_opt(Z_tos, aaddress(n));
1005 }
1006 
1007 void TemplateTable::aload_0() {
1008   aload_0_internal();
1009 }
1010 
1011 void TemplateTable::nofast_aload_0() {
1012   aload_0_internal(may_not_rewrite);
1013 }
1014 
1015 void TemplateTable::aload_0_internal(RewriteControl rc) {
1016   transition(vtos, atos);
1017 
1018   // According to bytecode histograms, the pairs:
1019   //
1020   // _aload_0, _fast_igetfield
1021   // _aload_0, _fast_agetfield
1022   // _aload_0, _fast_fgetfield
1023   //
1024   // occur frequently. If RewriteFrequentPairs is set, the (slow)
1025   // _aload_0 bytecode checks if the next bytecode is either
1026   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
1027   // rewrites the current bytecode into a pair bytecode; otherwise it
1028   // rewrites the current bytecode into _fast_aload_0 that doesn't do
1029   // the pair check anymore.
1030   //
1031   // Note: If the next bytecode is _getfield, the rewrite must be
1032   //       delayed, otherwise we may miss an opportunity for a pair.
1033   //
1034   // Also rewrite frequent pairs
1035   //   aload_0, aload_1
1036   //   aload_0, iload_1
1037   // These bytecodes with a small amount of code are most profitable
1038   // to rewrite.
1039   if (!(RewriteFrequentPairs && (rc == may_rewrite))) {
1040     aload(0);
1041     return;
1042   }
1043 
1044   NearLabel rewrite, done;
1045   const Register bc = Z_ARG4;
1046 
1047   assert(Z_R1_scratch != bc, "register damaged");
1048   // Get next byte.
1049   __ z_llgc(Z_R1_scratch, at_bcp(Bytecodes::length_for (Bytecodes::_aload_0)));
1050 
1051   // Do actual aload_0.
1052   aload(0);
1053 
1054   // If _getfield then wait with rewrite.
1055   __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_getfield,
1056                            Assembler::bcondEqual, done);
1057 
1058   // If _igetfield then rewrite to _fast_iaccess_0.
1059   assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0)
1060             == Bytecodes::_aload_0, "fix bytecode definition");
1061 
1062   __ load_const_optimized(bc, Bytecodes::_fast_iaccess_0);
1063   __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_fast_igetfield,
1064                            Assembler::bcondEqual, rewrite);
1065 
1066   // If _agetfield then rewrite to _fast_aaccess_0.
1067   assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0)
1068             == Bytecodes::_aload_0, "fix bytecode definition");
1069 
1070   __ load_const_optimized(bc, Bytecodes::_fast_aaccess_0);
1071   __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_fast_agetfield,
1072                            Assembler::bcondEqual, rewrite);
1073 
1074   // If _fgetfield then rewrite to _fast_faccess_0.
1075   assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0)
1076             == Bytecodes::_aload_0, "fix bytecode definition");
1077 
1078   __ load_const_optimized(bc, Bytecodes::_fast_faccess_0);
1079   __ compareU32_and_branch(Z_R1_scratch, Bytecodes::_fast_fgetfield,
1080                            Assembler::bcondEqual, rewrite);
1081 
1082   // Else rewrite to _fast_aload0.
1083   assert(Bytecodes::java_code(Bytecodes::_fast_aload_0)
1084             == Bytecodes::_aload_0, "fix bytecode definition");
1085   __ load_const_optimized(bc, Bytecodes::_fast_aload_0);
1086 
1087   // rewrite
1088   // bc: fast bytecode
1089   __ bind(rewrite);
1090 
1091   patch_bytecode(Bytecodes::_aload_0, bc, Z_R1_scratch, false);
1092   // Reload local 0 because of VM call inside patch_bytecode().
1093   // this may trigger GC and thus change the oop.
1094   aload(0);
1095 
1096   __ bind(done);
1097 }
1098 
1099 void TemplateTable::istore() {
1100   transition(itos, vtos);
1101   locals_index(Z_R1_scratch);
1102   __ reg2mem_opt(Z_tos, iaddress(_masm, Z_R1_scratch), false);
1103 }
1104 
1105 void TemplateTable::lstore() {
1106   transition(ltos, vtos);
1107   locals_index(Z_R1_scratch);
1108   __ reg2mem_opt(Z_tos, laddress(_masm, Z_R1_scratch));
1109 }
1110 
1111 void TemplateTable::fstore() {
1112   transition(ftos, vtos);
1113   locals_index(Z_R1_scratch);
1114   __ freg2mem_opt(Z_ftos, faddress(_masm, Z_R1_scratch));
1115 }
1116 
1117 void TemplateTable::dstore() {
1118   transition(dtos, vtos);
1119   locals_index(Z_R1_scratch);
1120   __ freg2mem_opt(Z_ftos, daddress(_masm, Z_R1_scratch));
1121 }
1122 
1123 void TemplateTable::astore() {
1124   transition(vtos, vtos);
1125   __ pop_ptr(Z_tos);
1126   locals_index(Z_R1_scratch);
1127   __ reg2mem_opt(Z_tos, aaddress(_masm, Z_R1_scratch));
1128 }
1129 
1130 void TemplateTable::wide_istore() {
1131   transition(vtos, vtos);
1132   __ pop_i(Z_tos);
1133   locals_index_wide(Z_tmp_1);
1134   __ reg2mem_opt(Z_tos, iaddress(_masm, Z_tmp_1), false);
1135 }
1136 
1137 void TemplateTable::wide_lstore() {
1138   transition(vtos, vtos);
1139   __ pop_l(Z_tos);
1140   locals_index_wide(Z_tmp_1);
1141   __ reg2mem_opt(Z_tos, laddress(_masm, Z_tmp_1));
1142 }
1143 
1144 void TemplateTable::wide_fstore() {
1145   transition(vtos, vtos);
1146   __ pop_f(Z_ftos);
1147   locals_index_wide(Z_tmp_1);
1148   __ freg2mem_opt(Z_ftos, faddress(_masm, Z_tmp_1), false);
1149 }
1150 
1151 void TemplateTable::wide_dstore() {
1152   transition(vtos, vtos);
1153   __ pop_d(Z_ftos);
1154   locals_index_wide(Z_tmp_1);
1155   __ freg2mem_opt(Z_ftos, daddress(_masm, Z_tmp_1));
1156 }
1157 
1158 void TemplateTable::wide_astore() {
1159   transition(vtos, vtos);
1160   __ pop_ptr(Z_tos);
1161   locals_index_wide(Z_tmp_1);
1162   __ reg2mem_opt(Z_tos, aaddress(_masm, Z_tmp_1));
1163 }
1164 
1165 void TemplateTable::iastore() {
1166   transition(itos, vtos);
1167 
1168   Register index = Z_ARG3; // Index_check expects index in Z_ARG3.
1169   // Value is in Z_tos ...
1170   __ pop_i(index);        // index
1171   __ pop_ptr(Z_tmp_1);    // array
1172   index_check(Z_tmp_1, index, LogBytesPerInt);
1173   // ... and then move the value.
1174   __ reg2mem_opt(Z_tos,
1175                  Address(Z_tmp_1, index, arrayOopDesc::base_offset_in_bytes(T_INT)),
1176                  false);
1177 }
1178 
1179 void TemplateTable::lastore() {
1180   transition(ltos, vtos);
1181 
1182   __ pop_i(Z_ARG3);
1183   __ pop_ptr(Z_tmp_2);
1184   // Z_tos   : value
1185   // Z_ARG3  : index
1186   // Z_tmp_2 : array
1187  index_check(Z_tmp_2, Z_ARG3, LogBytesPerLong); // Prefer index in Z_ARG3.
1188   __ reg2mem_opt(Z_tos,
1189                  Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_LONG)));
1190 }
1191 
1192 void TemplateTable::fastore() {
1193   transition(ftos, vtos);
1194 
1195   __ pop_i(Z_ARG3);
1196   __ pop_ptr(Z_tmp_2);
1197   // Z_ftos  : value
1198   // Z_ARG3  : index
1199   // Z_tmp_2 : array
1200   index_check(Z_tmp_2, Z_ARG3, LogBytesPerInt); // Prefer index in Z_ARG3.
1201   __ freg2mem_opt(Z_ftos,
1202                   Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_FLOAT)),
1203                   false);
1204 }
1205 
1206 void TemplateTable::dastore() {
1207   transition(dtos, vtos);
1208 
1209   __ pop_i(Z_ARG3);
1210   __ pop_ptr(Z_tmp_2);
1211   // Z_ftos  : value
1212   // Z_ARG3  : index
1213   // Z_tmp_2 : array
1214   index_check(Z_tmp_2, Z_ARG3, LogBytesPerLong); // Prefer index in Z_ARG3.
1215   __ freg2mem_opt(Z_ftos,
1216                   Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1217 }
1218 
1219 void TemplateTable::aastore() {
1220   NearLabel is_null, ok_is_subtype, done;
1221   transition(vtos, vtos);
1222 
1223   // stack: ..., array, index, value
1224 
1225   Register Rvalue = Z_tos;
1226   Register Rarray = Z_ARG2;
1227   Register Rindex = Z_ARG3; // Convention for index_check().
1228 
1229   __ load_ptr(0, Rvalue);
1230   __ z_l(Rindex, Address(Z_esp, Interpreter::expr_offset_in_bytes(1)));
1231   __ load_ptr(2, Rarray);
1232 
1233   unsigned const int shift = LogBytesPerHeapOop;
1234   index_check(Rarray, Rindex, shift); // side effect: Rindex = Rindex << shift
1235   Register Rstore_addr  = Rindex;
1236   // Address where the store goes to, i.e. &(Rarry[index])
1237   __ load_address(Rstore_addr, Address(Rarray, Rindex, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1238 
1239   // do array store check - check for NULL value first.
1240   __ compareU64_and_branch(Rvalue, (intptr_t)0, Assembler::bcondEqual, is_null);
1241 
1242   Register Rsub_klass   = Z_ARG4;
1243   Register Rsuper_klass = Z_ARG5;
1244   __ load_klass(Rsub_klass, Rvalue);
1245   // Load superklass.
1246   __ load_klass(Rsuper_klass, Rarray);
1247   __ z_lg(Rsuper_klass, Address(Rsuper_klass, ObjArrayKlass::element_klass_offset()));
1248 
1249   // Generate a fast subtype check.  Branch to ok_is_subtype if no failure.
1250   // Throw if failure.
1251   Register tmp1 = Z_tmp_1;
1252   Register tmp2 = Z_tmp_2;
1253   __ gen_subtype_check(Rsub_klass, Rsuper_klass, tmp1, tmp2, ok_is_subtype);
1254 
1255   // Fall through on failure.
1256   // Object is in Rvalue == Z_tos.
1257   assert(Rvalue == Z_tos, "that's the expected location");
1258   __ load_absolute_address(tmp1, Interpreter::_throw_ArrayStoreException_entry);
1259   __ z_br(tmp1);
1260 
1261   // Come here on success.
1262   __ bind(ok_is_subtype);
1263 
1264   // Now store using the appropriate barrier.
1265   Register tmp3 = Rsub_klass;
1266   do_oop_store(_masm, Rstore_addr, (intptr_t)0/*offset*/, Rvalue, false/*val==null*/,
1267                tmp3, tmp2, tmp1, _bs->kind(), true);
1268   __ z_bru(done);
1269 
1270   // Have a NULL in Rvalue.
1271   __ bind(is_null);
1272   __ profile_null_seen(tmp1);
1273 
1274   // Store a NULL.
1275   do_oop_store(_masm, Rstore_addr, (intptr_t)0/*offset*/, Rvalue, true/*val==null*/,
1276                tmp3, tmp2, tmp1, _bs->kind(), true);
1277 
1278   // Pop stack arguments.
1279   __ bind(done);
1280   __ add2reg(Z_esp, 3 * Interpreter::stackElementSize);
1281 }
1282 
1283 
1284 void TemplateTable::bastore() {
1285   transition(itos, vtos);
1286 
1287   __ pop_i(Z_ARG3);
1288   __ pop_ptr(Z_tmp_2);
1289   // Z_tos   : value
1290   // Z_ARG3  : index
1291   // Z_tmp_2 : array
1292 
1293   // Need to check whether array is boolean or byte
1294   // since both types share the bastore bytecode.
1295   __ load_klass(Z_tmp_1, Z_tmp_2);
1296   __ z_llgf(Z_tmp_1, Address(Z_tmp_1, Klass::layout_helper_offset()));
1297   __ z_tmll(Z_tmp_1, Klass::layout_helper_boolean_diffbit());
1298   Label L_skip;
1299   __ z_bfalse(L_skip);
1300   // if it is a T_BOOLEAN array, mask the stored value to 0/1
1301   __ z_nilf(Z_tos, 0x1);
1302   __ bind(L_skip);
1303 
1304   // No index shift necessary - pass 0.
1305   index_check(Z_tmp_2, Z_ARG3, 0); // Prefer index in Z_ARG3.
1306   __ z_stc(Z_tos,
1307            Address(Z_tmp_2, Z_ARG3, arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1308 }
1309 
1310 void TemplateTable::castore() {
1311   transition(itos, vtos);
1312 
1313   __ pop_i(Z_ARG3);
1314   __ pop_ptr(Z_tmp_2);
1315   // Z_tos   : value
1316   // Z_ARG3  : index
1317   // Z_tmp_2 : array
1318   Register index = Z_ARG3; // prefer index in Z_ARG3
1319   index_check(Z_tmp_2, index, LogBytesPerShort);
1320   __ z_sth(Z_tos,
1321            Address(Z_tmp_2, index, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1322 }
1323 
1324 void TemplateTable::sastore() {
1325   castore();
1326 }
1327 
1328 void TemplateTable::istore(int n) {
1329   transition(itos, vtos);
1330   __ reg2mem_opt(Z_tos, iaddress(n), false);
1331 }
1332 
1333 void TemplateTable::lstore(int n) {
1334   transition(ltos, vtos);
1335   __ reg2mem_opt(Z_tos, laddress(n));
1336 }
1337 
1338 void TemplateTable::fstore(int n) {
1339   transition(ftos, vtos);
1340   __ freg2mem_opt(Z_ftos, faddress(n), false);
1341 }
1342 
1343 void TemplateTable::dstore(int n) {
1344   transition(dtos, vtos);
1345   __ freg2mem_opt(Z_ftos, daddress(n));
1346 }
1347 
1348 void TemplateTable::astore(int n) {
1349   transition(vtos, vtos);
1350   __ pop_ptr(Z_tos);
1351   __ reg2mem_opt(Z_tos, aaddress(n));
1352 }
1353 
1354 void TemplateTable::pop() {
1355   transition(vtos, vtos);
1356   __ add2reg(Z_esp, Interpreter::stackElementSize);
1357 }
1358 
1359 void TemplateTable::pop2() {
1360   transition(vtos, vtos);
1361   __ add2reg(Z_esp, 2 * Interpreter::stackElementSize);
1362 }
1363 
1364 void TemplateTable::dup() {
1365   transition(vtos, vtos);
1366   __ load_ptr(0, Z_tos);
1367   __ push_ptr(Z_tos);
1368   // stack: ..., a, a
1369 }
1370 
1371 void TemplateTable::dup_x1() {
1372   transition(vtos, vtos);
1373 
1374   // stack: ..., a, b
1375   __ load_ptr(0, Z_tos);          // load b
1376   __ load_ptr(1, Z_R0_scratch);   // load a
1377   __ store_ptr(1, Z_tos);         // store b
1378   __ store_ptr(0, Z_R0_scratch);  // store a
1379   __ push_ptr(Z_tos);             // push b
1380   // stack: ..., b, a, b
1381 }
1382 
1383 void TemplateTable::dup_x2() {
1384   transition(vtos, vtos);
1385 
1386   // stack: ..., a, b, c
1387   __ load_ptr(0, Z_R0_scratch);   // load c
1388   __ load_ptr(2, Z_R1_scratch);   // load a
1389   __ store_ptr(2, Z_R0_scratch);  // store c in a
1390   __ push_ptr(Z_R0_scratch);      // push c
1391   // stack: ..., c, b, c, c
1392   __ load_ptr(2, Z_R0_scratch);   // load b
1393   __ store_ptr(2, Z_R1_scratch);  // store a in b
1394   // stack: ..., c, a, c, c
1395   __ store_ptr(1, Z_R0_scratch);  // store b in c
1396   // stack: ..., c, a, b, c
1397 }
1398 
1399 void TemplateTable::dup2() {
1400   transition(vtos, vtos);
1401 
1402   // stack: ..., a, b
1403   __ load_ptr(1, Z_R0_scratch);  // load a
1404   __ push_ptr(Z_R0_scratch);     // push a
1405   __ load_ptr(1, Z_R0_scratch);  // load b
1406   __ push_ptr(Z_R0_scratch);     // push b
1407   // stack: ..., a, b, a, b
1408 }
1409 
1410 void TemplateTable::dup2_x1() {
1411   transition(vtos, vtos);
1412 
1413   // stack: ..., a, b, c
1414   __ load_ptr(0, Z_R0_scratch);  // load c
1415   __ load_ptr(1, Z_R1_scratch);  // load b
1416   __ push_ptr(Z_R1_scratch);     // push b
1417   __ push_ptr(Z_R0_scratch);     // push c
1418   // stack: ..., a, b, c, b, c
1419   __ store_ptr(3, Z_R0_scratch); // store c in b
1420   // stack: ..., a, c, c, b, c
1421   __ load_ptr( 4, Z_R0_scratch); // load a
1422   __ store_ptr(2, Z_R0_scratch); // store a in 2nd c
1423   // stack: ..., a, c, a, b, c
1424   __ store_ptr(4, Z_R1_scratch); // store b in a
1425   // stack: ..., b, c, a, b, c
1426 }
1427 
1428 void TemplateTable::dup2_x2() {
1429   transition(vtos, vtos);
1430 
1431   // stack: ..., a, b, c, d
1432   __ load_ptr(0, Z_R0_scratch);   // load d
1433   __ load_ptr(1, Z_R1_scratch);   // load c
1434   __ push_ptr(Z_R1_scratch);      // push c
1435   __ push_ptr(Z_R0_scratch);      // push d
1436   // stack: ..., a, b, c, d, c, d
1437   __ load_ptr(4, Z_R1_scratch);   // load b
1438   __ store_ptr(2, Z_R1_scratch);  // store b in d
1439   __ store_ptr(4, Z_R0_scratch);  // store d in b
1440   // stack: ..., a, d, c, b, c, d
1441   __ load_ptr(5, Z_R0_scratch);   // load a
1442   __ load_ptr(3, Z_R1_scratch);   // load c
1443   __ store_ptr(3, Z_R0_scratch);  // store a in c
1444   __ store_ptr(5, Z_R1_scratch);  // store c in a
1445   // stack: ..., c, d, a, b, c, d
1446 }
1447 
1448 void TemplateTable::swap() {
1449   transition(vtos, vtos);
1450 
1451   // stack: ..., a, b
1452   __ load_ptr(1, Z_R0_scratch);  // load a
1453   __ load_ptr(0, Z_R1_scratch);  // load b
1454   __ store_ptr(0, Z_R0_scratch);  // store a in b
1455   __ store_ptr(1, Z_R1_scratch);  // store b in a
1456   // stack: ..., b, a
1457 }
1458 
1459 void TemplateTable::iop2(Operation op) {
1460   transition(itos, itos);
1461   switch (op) {
1462     case add  :                           __ z_ay(Z_tos,  __ stackTop()); __ pop_i(); break;
1463     case sub  :                           __ z_sy(Z_tos,  __ stackTop()); __ pop_i(); __ z_lcr(Z_tos, Z_tos); break;
1464     case mul  :                           __ z_msy(Z_tos, __ stackTop()); __ pop_i(); break;
1465     case _and :                           __ z_ny(Z_tos,  __ stackTop()); __ pop_i(); break;
1466     case _or  :                           __ z_oy(Z_tos,  __ stackTop()); __ pop_i(); break;
1467     case _xor :                           __ z_xy(Z_tos,  __ stackTop()); __ pop_i(); break;
1468     case shl  : __ z_lr(Z_tmp_1, Z_tos);
1469                 __ z_nill(Z_tmp_1, 31);  // Lowest 5 bits are shiftamount.
1470                                           __ pop_i(Z_tos);   __ z_sll(Z_tos, 0,  Z_tmp_1); break;
1471     case shr  : __ z_lr(Z_tmp_1, Z_tos);
1472                 __ z_nill(Z_tmp_1, 31);  // Lowest 5 bits are shiftamount.
1473                                           __ pop_i(Z_tos);   __ z_sra(Z_tos, 0,  Z_tmp_1); break;
1474     case ushr : __ z_lr(Z_tmp_1, Z_tos);
1475                 __ z_nill(Z_tmp_1, 31);  // Lowest 5 bits are shiftamount.
1476                                           __ pop_i(Z_tos);   __ z_srl(Z_tos, 0,  Z_tmp_1); break;
1477     default   : ShouldNotReachHere(); break;
1478   }
1479   return;
1480 }
1481 
1482 void TemplateTable::lop2(Operation op) {
1483   transition(ltos, ltos);
1484 
1485   switch (op) {
1486     case add  :  __ z_ag(Z_tos,  __ stackTop()); __ pop_l(); break;
1487     case sub  :  __ z_sg(Z_tos,  __ stackTop()); __ pop_l(); __ z_lcgr(Z_tos, Z_tos); break;
1488     case mul  :  __ z_msg(Z_tos, __ stackTop()); __ pop_l(); break;
1489     case _and :  __ z_ng(Z_tos,  __ stackTop()); __ pop_l(); break;
1490     case _or  :  __ z_og(Z_tos,  __ stackTop()); __ pop_l(); break;
1491     case _xor :  __ z_xg(Z_tos,  __ stackTop()); __ pop_l(); break;
1492     default   : ShouldNotReachHere(); break;
1493   }
1494   return;
1495 }
1496 
1497 // Common part of idiv/irem.
1498 static void idiv_helper(InterpreterMacroAssembler * _masm, address exception) {
1499   NearLabel not_null;
1500 
1501   // Use register pair Z_tmp_1, Z_tmp_2 for DIVIDE SINGLE.
1502   assert(Z_tmp_1->successor() == Z_tmp_2, " need even/odd register pair for idiv/irem");
1503 
1504   // Get dividend.
1505   __ pop_i(Z_tmp_2);
1506 
1507   // If divisor == 0 throw exception.
1508   __ compare32_and_branch(Z_tos, (intptr_t) 0,
1509                           Assembler::bcondNotEqual, not_null   );
1510   __ load_absolute_address(Z_R1_scratch, exception);
1511   __ z_br(Z_R1_scratch);
1512 
1513   __ bind(not_null);
1514 
1515   __ z_lgfr(Z_tmp_2, Z_tmp_2);   // Sign extend dividend.
1516   __ z_dsgfr(Z_tmp_1, Z_tos);    // Do it.
1517 }
1518 
1519 void TemplateTable::idiv() {
1520   transition(itos, itos);
1521 
1522   idiv_helper(_masm, Interpreter::_throw_ArithmeticException_entry);
1523   __ z_llgfr(Z_tos, Z_tmp_2);     // Result is in Z_tmp_2.
1524 }
1525 
1526 void TemplateTable::irem() {
1527   transition(itos, itos);
1528 
1529   idiv_helper(_masm, Interpreter::_throw_ArithmeticException_entry);
1530   __ z_llgfr(Z_tos, Z_tmp_1);     // Result is in Z_tmp_1.
1531 }
1532 
1533 void TemplateTable::lmul() {
1534   transition(ltos, ltos);
1535 
1536   // Multiply with memory operand.
1537   __ z_msg(Z_tos, __ stackTop());
1538   __ pop_l();  // Pop operand.
1539 }
1540 
1541 // Common part of ldiv/lrem.
1542 //
1543 // Input:
1544 //     Z_tos := the divisor (dividend still on stack)
1545 //
1546 // Updated registers:
1547 //     Z_tmp_1 := pop_l() % Z_tos     ; if is_ldiv == false
1548 //     Z_tmp_2 := pop_l() / Z_tos     ; if is_ldiv == true
1549 //
1550 static void ldiv_helper(InterpreterMacroAssembler * _masm, address exception, bool is_ldiv) {
1551   NearLabel not_null, done;
1552 
1553   // Use register pair Z_tmp_1, Z_tmp_2 for DIVIDE SINGLE.
1554   assert(Z_tmp_1->successor() == Z_tmp_2,
1555          " need even/odd register pair for idiv/irem");
1556 
1557   // Get dividend.
1558   __ pop_l(Z_tmp_2);
1559 
1560   // If divisor == 0 throw exception.
1561   __ compare64_and_branch(Z_tos, (intptr_t)0, Assembler::bcondNotEqual, not_null);
1562   __ load_absolute_address(Z_R1_scratch, exception);
1563   __ z_br(Z_R1_scratch);
1564 
1565   __ bind(not_null);
1566   // Special case for dividend == 0x8000 and divisor == -1.
1567   if (is_ldiv) {
1568     // result := Z_tmp_2 := - dividend
1569     __ z_lcgr(Z_tmp_2, Z_tmp_2);
1570   } else {
1571     // result remainder := Z_tmp_1 := 0
1572     __ clear_reg(Z_tmp_1, true, false);  // Don't set CC.
1573   }
1574 
1575   // if divisor == -1 goto done
1576   __ compare64_and_branch(Z_tos, -1, Assembler::bcondEqual, done);
1577   if (is_ldiv)
1578     // Restore sign, because divisor != -1.
1579     __ z_lcgr(Z_tmp_2, Z_tmp_2);
1580   __ z_dsgr(Z_tmp_1, Z_tos);    // Do it.
1581   __ bind(done);
1582 }
1583 
1584 void TemplateTable::ldiv() {
1585   transition(ltos, ltos);
1586 
1587   ldiv_helper(_masm, Interpreter::_throw_ArithmeticException_entry, true /*is_ldiv*/);
1588   __ z_lgr(Z_tos, Z_tmp_2);     // Result is in Z_tmp_2.
1589 }
1590 
1591 void TemplateTable::lrem() {
1592   transition(ltos, ltos);
1593 
1594   ldiv_helper(_masm, Interpreter::_throw_ArithmeticException_entry, false /*is_ldiv*/);
1595   __ z_lgr(Z_tos, Z_tmp_1);     // Result is in Z_tmp_1.
1596 }
1597 
1598 void TemplateTable::lshl() {
1599   transition(itos, ltos);
1600 
1601   // Z_tos: shift amount
1602   __ pop_l(Z_tmp_1);              // Get shift value.
1603   __ z_sllg(Z_tos, Z_tmp_1, 0, Z_tos);
1604 }
1605 
1606 void TemplateTable::lshr() {
1607   transition(itos, ltos);
1608 
1609   // Z_tos: shift amount
1610   __ pop_l(Z_tmp_1);              // Get shift value.
1611   __ z_srag(Z_tos, Z_tmp_1, 0, Z_tos);
1612 }
1613 
1614 void TemplateTable::lushr() {
1615   transition(itos, ltos);
1616 
1617   // Z_tos: shift amount
1618   __ pop_l(Z_tmp_1);              // Get shift value.
1619   __ z_srlg(Z_tos, Z_tmp_1, 0, Z_tos);
1620 }
1621 
1622 void TemplateTable::fop2(Operation op) {
1623   transition(ftos, ftos);
1624 
1625   switch (op) {
1626     case add:
1627       // Add memory operand.
1628       __ z_aeb(Z_ftos, __ stackTop()); __ pop_f(); return;
1629     case sub:
1630       // Sub memory operand.
1631       __ z_ler(Z_F1, Z_ftos);    // first operand
1632       __ pop_f(Z_ftos);          // second operand from stack
1633       __ z_sebr(Z_ftos, Z_F1);
1634       return;
1635     case mul:
1636       // Multiply with memory operand.
1637       __ z_meeb(Z_ftos, __ stackTop()); __ pop_f(); return;
1638     case div:
1639       __ z_ler(Z_F1, Z_ftos);    // first operand
1640       __ pop_f(Z_ftos);          // second operand from stack
1641       __ z_debr(Z_ftos, Z_F1);
1642       return;
1643     case rem:
1644       // Do runtime call.
1645       __ z_ler(Z_FARG2, Z_ftos);  // divisor
1646       __ pop_f(Z_FARG1);          // dividend
1647       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1648       // Result should be in the right place (Z_ftos == Z_FRET).
1649       return;
1650     default:
1651       ShouldNotReachHere();
1652       return;
1653   }
1654 }
1655 
1656 void TemplateTable::dop2(Operation op) {
1657   transition(dtos, dtos);
1658 
1659   switch (op) {
1660     case add:
1661       // Add memory operand.
1662       __ z_adb(Z_ftos, __ stackTop()); __ pop_d(); return;
1663     case sub:
1664       // Sub memory operand.
1665       __ z_ldr(Z_F1, Z_ftos);    // first operand
1666       __ pop_d(Z_ftos);          // second operand from stack
1667       __ z_sdbr(Z_ftos, Z_F1);
1668       return;
1669     case mul:
1670       // Multiply with memory operand.
1671       __ z_mdb(Z_ftos, __ stackTop()); __ pop_d(); return;
1672     case div:
1673       __ z_ldr(Z_F1, Z_ftos);    // first operand
1674       __ pop_d(Z_ftos);          // second operand from stack
1675       __ z_ddbr(Z_ftos, Z_F1);
1676       return;
1677     case rem:
1678       // Do runtime call.
1679       __ z_ldr(Z_FARG2, Z_ftos);  // divisor
1680       __ pop_d(Z_FARG1);          // dividend
1681       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1682       // Result should be in the right place (Z_ftos == Z_FRET).
1683       return;
1684     default:
1685       ShouldNotReachHere();
1686       return;
1687   }
1688 }
1689 
1690 void TemplateTable::ineg() {
1691   transition(itos, itos);
1692   __ z_lcr(Z_tos);
1693 }
1694 
1695 void TemplateTable::lneg() {
1696   transition(ltos, ltos);
1697   __ z_lcgr(Z_tos);
1698 }
1699 
1700 void TemplateTable::fneg() {
1701   transition(ftos, ftos);
1702   __ z_lcebr(Z_ftos, Z_ftos);
1703 }
1704 
1705 void TemplateTable::dneg() {
1706   transition(dtos, dtos);
1707   __ z_lcdbr(Z_ftos, Z_ftos);
1708 }
1709 
1710 void TemplateTable::iinc() {
1711   transition(vtos, vtos);
1712 
1713   Address local;
1714   __ z_lb(Z_R0_scratch, at_bcp(2)); // Get constant.
1715   locals_index(Z_R1_scratch);
1716   local = iaddress(_masm, Z_R1_scratch);
1717   __ z_a(Z_R0_scratch, local);
1718   __ reg2mem_opt(Z_R0_scratch, local, false);
1719 }
1720 
1721 void TemplateTable::wide_iinc() {
1722   transition(vtos, vtos);
1723 
1724   // Z_tmp_1 := increment
1725   __ get_2_byte_integer_at_bcp(Z_tmp_1, 4, InterpreterMacroAssembler::Signed);
1726   // Z_R1_scratch := index of local to increment
1727   locals_index_wide(Z_tmp_2);
1728   // Load, increment, and store.
1729   __ access_local_int(Z_tmp_2, Z_tos);
1730   __ z_agr(Z_tos,  Z_tmp_1);
1731   // Shifted index is still in Z_tmp_2.
1732   __ reg2mem_opt(Z_tos, Address(Z_locals, Z_tmp_2), false);
1733 }
1734 
1735 
1736 void TemplateTable::convert() {
1737   // Checking
1738 #ifdef ASSERT
1739   TosState   tos_in  = ilgl;
1740   TosState   tos_out = ilgl;
1741 
1742   switch (bytecode()) {
1743     case Bytecodes::_i2l:
1744     case Bytecodes::_i2f:
1745     case Bytecodes::_i2d:
1746     case Bytecodes::_i2b:
1747     case Bytecodes::_i2c:
1748     case Bytecodes::_i2s:
1749       tos_in = itos;
1750       break;
1751     case Bytecodes::_l2i:
1752     case Bytecodes::_l2f:
1753     case Bytecodes::_l2d:
1754       tos_in = ltos;
1755       break;
1756     case Bytecodes::_f2i:
1757     case Bytecodes::_f2l:
1758     case Bytecodes::_f2d:
1759       tos_in = ftos;
1760       break;
1761     case Bytecodes::_d2i:
1762     case Bytecodes::_d2l:
1763     case Bytecodes::_d2f:
1764       tos_in = dtos;
1765       break;
1766     default :
1767       ShouldNotReachHere();
1768   }
1769   switch (bytecode()) {
1770     case Bytecodes::_l2i:
1771     case Bytecodes::_f2i:
1772     case Bytecodes::_d2i:
1773     case Bytecodes::_i2b:
1774     case Bytecodes::_i2c:
1775     case Bytecodes::_i2s:
1776       tos_out = itos;
1777       break;
1778     case Bytecodes::_i2l:
1779     case Bytecodes::_f2l:
1780     case Bytecodes::_d2l:
1781       tos_out = ltos;
1782       break;
1783     case Bytecodes::_i2f:
1784     case Bytecodes::_l2f:
1785     case Bytecodes::_d2f:
1786       tos_out = ftos;
1787       break;
1788     case Bytecodes::_i2d:
1789     case Bytecodes::_l2d:
1790     case Bytecodes::_f2d:
1791       tos_out = dtos;
1792       break;
1793     default :
1794       ShouldNotReachHere();
1795   }
1796 
1797   transition(tos_in, tos_out);
1798 #endif // ASSERT
1799 
1800   // Conversion
1801   Label done;
1802   switch (bytecode()) {
1803     case Bytecodes::_i2l:
1804       __ z_lgfr(Z_tos, Z_tos);
1805       return;
1806     case Bytecodes::_i2f:
1807       __ z_cefbr(Z_ftos, Z_tos);
1808       return;
1809     case Bytecodes::_i2d:
1810       __ z_cdfbr(Z_ftos, Z_tos);
1811       return;
1812     case Bytecodes::_i2b:
1813       // Sign extend least significant byte.
1814       __ move_reg_if_needed(Z_tos, T_BYTE, Z_tos, T_INT);
1815       return;
1816     case Bytecodes::_i2c:
1817       // Zero extend 2 least significant bytes.
1818       __ move_reg_if_needed(Z_tos, T_CHAR, Z_tos, T_INT);
1819       return;
1820     case Bytecodes::_i2s:
1821       // Sign extend 2 least significant bytes.
1822       __ move_reg_if_needed(Z_tos, T_SHORT, Z_tos, T_INT);
1823       return;
1824     case Bytecodes::_l2i:
1825       // Sign-extend not needed here, upper 4 bytes of int value in register are ignored.
1826       return;
1827     case Bytecodes::_l2f:
1828       __ z_cegbr(Z_ftos, Z_tos);
1829       return;
1830     case Bytecodes::_l2d:
1831       __ z_cdgbr(Z_ftos, Z_tos);
1832       return;
1833     case Bytecodes::_f2i:
1834     case Bytecodes::_f2l:
1835       __ clear_reg(Z_tos, true, false);  // Don't set CC.
1836       __ z_cebr(Z_ftos, Z_ftos);
1837       __ z_brno(done); // NaN -> 0
1838       if (bytecode() == Bytecodes::_f2i)
1839         __ z_cfebr(Z_tos, Z_ftos, Assembler::to_zero);
1840       else // bytecode() == Bytecodes::_f2l
1841         __ z_cgebr(Z_tos, Z_ftos, Assembler::to_zero);
1842       break;
1843     case Bytecodes::_f2d:
1844       __ move_freg_if_needed(Z_ftos, T_DOUBLE, Z_ftos, T_FLOAT);
1845       return;
1846     case Bytecodes::_d2i:
1847     case Bytecodes::_d2l:
1848       __ clear_reg(Z_tos, true, false);  // Ddon't set CC.
1849       __ z_cdbr(Z_ftos, Z_ftos);
1850       __ z_brno(done); // NaN -> 0
1851       if (bytecode() == Bytecodes::_d2i)
1852         __ z_cfdbr(Z_tos, Z_ftos, Assembler::to_zero);
1853       else // Bytecodes::_d2l
1854         __ z_cgdbr(Z_tos, Z_ftos, Assembler::to_zero);
1855       break;
1856     case Bytecodes::_d2f:
1857       __ move_freg_if_needed(Z_ftos, T_FLOAT, Z_ftos, T_DOUBLE);
1858       return;
1859     default:
1860       ShouldNotReachHere();
1861   }
1862   __ bind(done);
1863 }
1864 
1865 void TemplateTable::lcmp() {
1866   transition(ltos, itos);
1867 
1868   Label   done;
1869   Register val1 = Z_R0_scratch;
1870   Register val2 = Z_R1_scratch;
1871 
1872   if (VM_Version::has_LoadStoreConditional()) {
1873     __ pop_l(val1);           // pop value 1.
1874     __ z_lghi(val2,  -1);     // lt value
1875     __ z_cgr(val1, Z_tos);    // Compare with Z_tos (value 2). Protect CC under all circumstances.
1876     __ z_lghi(val1,   1);     // gt value
1877     __ z_lghi(Z_tos,  0);     // eq value
1878 
1879     __ z_locgr(Z_tos, val1, Assembler::bcondHigh);
1880     __ z_locgr(Z_tos, val2, Assembler::bcondLow);
1881   } else {
1882     __ pop_l(val1);           // Pop value 1.
1883     __ z_cgr(val1, Z_tos);    // Compare with Z_tos (value 2). Protect CC under all circumstances.
1884 
1885     __ z_lghi(Z_tos,  0);     // eq value
1886     __ z_bre(done);
1887 
1888     __ z_lghi(Z_tos,  1);     // gt value
1889     __ z_brh(done);
1890 
1891     __ z_lghi(Z_tos, -1);     // lt value
1892   }
1893 
1894   __ bind(done);
1895 }
1896 
1897 
1898 void TemplateTable::float_cmp(bool is_float, int unordered_result) {
1899   Label done;
1900 
1901   if (is_float) {
1902     __ pop_f(Z_FARG2);
1903     __ z_cebr(Z_FARG2, Z_ftos);
1904   } else {
1905     __ pop_d(Z_FARG2);
1906     __ z_cdbr(Z_FARG2, Z_ftos);
1907   }
1908 
1909   if (VM_Version::has_LoadStoreConditional()) {
1910     Register one       = Z_R0_scratch;
1911     Register minus_one = Z_R1_scratch;
1912     __ z_lghi(minus_one,  -1);
1913     __ z_lghi(one,  1);
1914     __ z_lghi(Z_tos, 0);
1915     __ z_locgr(Z_tos, one,       unordered_result == 1 ? Assembler::bcondHighOrNotOrdered : Assembler::bcondHigh);
1916     __ z_locgr(Z_tos, minus_one, unordered_result == 1 ? Assembler::bcondLow              : Assembler::bcondLowOrNotOrdered);
1917   } else {
1918     // Z_FARG2 == Z_ftos
1919     __ clear_reg(Z_tos, false, false);
1920     __ z_bre(done);
1921 
1922     // F_ARG2 > Z_Ftos, or unordered
1923     __ z_lhi(Z_tos, 1);
1924     __ z_brc(unordered_result == 1 ? Assembler::bcondHighOrNotOrdered : Assembler::bcondHigh, done);
1925 
1926     // F_ARG2 < Z_FTOS, or unordered
1927     __ z_lhi(Z_tos, -1);
1928 
1929     __ bind(done);
1930   }
1931 }
1932 
1933 void TemplateTable::branch(bool is_jsr, bool is_wide) {
1934   const Register   bumped_count = Z_tmp_1;
1935   const Register   method       = Z_tmp_2;
1936   const Register   m_counters   = Z_R1_scratch;
1937   const Register   mdo          = Z_tos;
1938 
1939   BLOCK_COMMENT("TemplateTable::branch {");
1940   __ get_method(method);
1941   __ profile_taken_branch(mdo, bumped_count);
1942 
1943   const ByteSize ctr_offset = InvocationCounter::counter_offset();
1944   const ByteSize be_offset  = MethodCounters::backedge_counter_offset()   + ctr_offset;
1945   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() + ctr_offset;
1946 
1947   // Get (wide) offset to disp.
1948   const Register disp = Z_ARG5;
1949   if (is_wide) {
1950     __ get_4_byte_integer_at_bcp(disp, 1);
1951   } else {
1952     __ get_2_byte_integer_at_bcp(disp, 1, InterpreterMacroAssembler::Signed);
1953   }
1954 
1955   // Handle all the JSR stuff here, then exit.
1956   // It's much shorter and cleaner than intermingling with the
1957   // non-JSR normal-branch stuff occurring below.
1958   if (is_jsr) {
1959     // Compute return address as bci in Z_tos.
1960     __ z_lgr(Z_R1_scratch, Z_bcp);
1961     __ z_sg(Z_R1_scratch, Address(method, Method::const_offset()));
1962     __ add2reg(Z_tos, (is_wide ? 5 : 3) - in_bytes(ConstMethod::codes_offset()), Z_R1_scratch);
1963 
1964     // Bump bcp to target of JSR.
1965     __ z_agr(Z_bcp, disp);
1966     // Push return address for "ret" on stack.
1967     __ push_ptr(Z_tos);
1968     // And away we go!
1969     __ dispatch_next(vtos, 0 , true);
1970     return;
1971   }
1972 
1973   // Normal (non-jsr) branch handling.
1974 
1975   // Bump bytecode pointer by displacement (take the branch).
1976   __ z_agr(Z_bcp, disp);
1977 
1978   assert(UseLoopCounter || !UseOnStackReplacement,
1979          "on-stack-replacement requires loop counters");
1980 
1981   NearLabel backedge_counter_overflow;
1982   NearLabel profile_method;
1983   NearLabel dispatch;
1984   int       increment = InvocationCounter::count_increment;
1985 
1986   if (UseLoopCounter) {
1987     // Increment backedge counter for backward branches.
1988     // disp: target offset
1989     // Z_bcp: target bcp
1990     // Z_locals: locals pointer
1991     //
1992     // Count only if backward branch.
1993     __ compare32_and_branch(disp, (intptr_t)0, Assembler::bcondHigh, dispatch);
1994 
1995     if (TieredCompilation) {
1996       Label noCounters;
1997 
1998       if (ProfileInterpreter) {
1999         NearLabel   no_mdo;
2000 
2001         // Are we profiling?
2002         __ load_and_test_long(mdo, Address(method, Method::method_data_offset()));
2003         __ branch_optimized(Assembler::bcondZero, no_mdo);
2004 
2005         // Increment the MDO backedge counter.
2006         const Address mdo_backedge_counter(mdo, MethodData::backedge_counter_offset() + InvocationCounter::counter_offset());
2007 
2008         const Address mask(mdo, MethodData::backedge_mask_offset());
2009         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
2010                                    Z_ARG2, false, Assembler::bcondZero,
2011                                    UseOnStackReplacement ? &backedge_counter_overflow : NULL);
2012         __ z_bru(dispatch);
2013         __ bind(no_mdo);
2014       }
2015 
2016       // Increment backedge counter in MethodCounters*.
2017       __ get_method_counters(method, m_counters, noCounters);
2018       const Address mask(m_counters, MethodCounters::backedge_mask_offset());
2019       __ increment_mask_and_jump(Address(m_counters, be_offset),
2020                                  increment, mask,
2021                                  Z_ARG2, false, Assembler::bcondZero,
2022                                  UseOnStackReplacement ? &backedge_counter_overflow : NULL);
2023       __ bind(noCounters);
2024     } else {
2025       Register counter = Z_tos;
2026       Label    noCounters;
2027       // Get address of MethodCounters object.
2028       __ get_method_counters(method, m_counters, noCounters);
2029       // Increment backedge counter.
2030       __ increment_backedge_counter(m_counters, counter);
2031 
2032       if (ProfileInterpreter) {
2033         // Test to see if we should create a method data obj.
2034         __ z_cl(counter, Address(m_counters, MethodCounters::interpreter_profile_limit_offset()));
2035         __ z_brl(dispatch);
2036 
2037         // If no method data exists, go to profile method.
2038         __ test_method_data_pointer(Z_ARG4/*result unused*/, profile_method);
2039 
2040         if (UseOnStackReplacement) {
2041           // Check for overflow against 'bumped_count' which is the MDO taken count.
2042           __ z_cl(bumped_count, Address(m_counters, MethodCounters::interpreter_backward_branch_limit_offset()));
2043           __ z_brl(dispatch);
2044 
2045           // When ProfileInterpreter is on, the backedge_count comes
2046           // from the methodDataOop, which value does not get reset on
2047           // the call to frequency_counter_overflow(). To avoid
2048           // excessive calls to the overflow routine while the method is
2049           // being compiled, add a second test to make sure the overflow
2050           // function is called only once every overflow_frequency.
2051           const int overflow_frequency = 1024;
2052           __ and_imm(bumped_count, overflow_frequency - 1);
2053           __ z_brz(backedge_counter_overflow);
2054 
2055         }
2056       } else {
2057         if (UseOnStackReplacement) {
2058           // Check for overflow against 'counter', which is the sum of the
2059           // counters.
2060           __ z_cl(counter, Address(m_counters, MethodCounters::interpreter_backward_branch_limit_offset()));
2061           __ z_brh(backedge_counter_overflow);
2062         }
2063       }
2064       __ bind(noCounters);
2065     }
2066 
2067     __ bind(dispatch);
2068   }
2069 
2070   // Pre-load the next target bytecode into rbx.
2071   __ z_llgc(Z_bytecode, Address(Z_bcp, (intptr_t) 0));
2072 
2073   // Continue with the bytecode @ target.
2074   // Z_tos: Return bci for jsr's, unused otherwise.
2075   // Z_bytecode: target bytecode
2076   // Z_bcp: target bcp
2077   __ dispatch_only(vtos, true);
2078 
2079   // Out-of-line code runtime calls.
2080   if (UseLoopCounter) {
2081     if (ProfileInterpreter) {
2082       // Out-of-line code to allocate method data oop.
2083       __ bind(profile_method);
2084 
2085       __ call_VM(noreg,
2086                  CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
2087       __ z_llgc(Z_bytecode, Address(Z_bcp, (intptr_t) 0));  // Restore target bytecode.
2088       __ set_method_data_pointer_for_bcp();
2089       __ z_bru(dispatch);
2090     }
2091 
2092     if (UseOnStackReplacement) {
2093 
2094       // invocation counter overflow
2095       __ bind(backedge_counter_overflow);
2096 
2097       __ z_lcgr(Z_ARG2, disp); // Z_ARG2 := -disp
2098       __ z_agr(Z_ARG2, Z_bcp); // Z_ARG2 := branch target bcp - disp == branch bcp
2099       __ call_VM(noreg,
2100                  CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow),
2101                  Z_ARG2);
2102 
2103       // Z_RET: osr nmethod (osr ok) or NULL (osr not possible).
2104       __ compare64_and_branch(Z_RET, (intptr_t) 0, Assembler::bcondEqual, dispatch);
2105 
2106       // Nmethod may have been invalidated (VM may block upon call_VM return).
2107       __ z_cliy(nmethod::state_offset(), Z_RET, nmethod::in_use);
2108       __ z_brne(dispatch);
2109 
2110       // Migrate the interpreter frame off of the stack.
2111 
2112       __ z_lgr(Z_tmp_1, Z_RET); // Save the nmethod.
2113 
2114       call_VM(noreg,
2115               CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2116 
2117       // Z_RET is OSR buffer, move it to expected parameter location.
2118       __ lgr_if_needed(Z_ARG1, Z_RET);
2119 
2120       // Pop the interpreter frame ...
2121       __ pop_interpreter_frame(Z_R14, Z_ARG2/*tmp1*/, Z_ARG3/*tmp2*/);
2122 
2123       // ... and begin the OSR nmethod.
2124       __ z_lg(Z_R1_scratch, Address(Z_tmp_1, nmethod::osr_entry_point_offset()));
2125       __ z_br(Z_R1_scratch);
2126     }
2127   }
2128   BLOCK_COMMENT("} TemplateTable::branch");
2129 }
2130 
2131 void TemplateTable::if_0cmp(Condition cc) {
2132   transition(itos, vtos);
2133 
2134   // Assume branch is more often taken than not (loops use backward branches).
2135   NearLabel not_taken;
2136   __ compare32_and_branch(Z_tos, (intptr_t) 0, j_not(cc), not_taken);
2137   branch(false, false);
2138   __ bind(not_taken);
2139   __ profile_not_taken_branch(Z_tos);
2140 }
2141 
2142 void TemplateTable::if_icmp(Condition cc) {
2143   transition(itos, vtos);
2144 
2145   // Assume branch is more often taken than not (loops use backward branches).
2146   NearLabel not_taken;
2147   __ pop_i(Z_R0_scratch);
2148   __ compare32_and_branch(Z_R0_scratch, Z_tos, j_not(cc), not_taken);
2149   branch(false, false);
2150   __ bind(not_taken);
2151   __ profile_not_taken_branch(Z_tos);
2152 }
2153 
2154 void TemplateTable::if_nullcmp(Condition cc) {
2155   transition(atos, vtos);
2156 
2157   // Assume branch is more often taken than not (loops use backward branches) .
2158   NearLabel not_taken;
2159   __ compare64_and_branch(Z_tos, (intptr_t) 0, j_not(cc), not_taken);
2160   branch(false, false);
2161   __ bind(not_taken);
2162   __ profile_not_taken_branch(Z_tos);
2163 }
2164 
2165 void TemplateTable::if_acmp(Condition cc) {
2166   transition(atos, vtos);
2167   // Assume branch is more often taken than not (loops use backward branches).
2168   NearLabel not_taken;
2169   __ pop_ptr(Z_ARG2);
2170   __ verify_oop(Z_ARG2);
2171   __ verify_oop(Z_tos);
2172   __ compareU64_and_branch(Z_tos, Z_ARG2, j_not(cc), not_taken);
2173   branch(false, false);
2174   __ bind(not_taken);
2175   __ profile_not_taken_branch(Z_ARG3);
2176 }
2177 
2178 void TemplateTable::ret() {
2179   transition(vtos, vtos);
2180 
2181   locals_index(Z_tmp_1);
2182   // Get return bci, compute return bcp. Must load 64 bits.
2183   __ mem2reg_opt(Z_tmp_1, iaddress(_masm, Z_tmp_1));
2184   __ profile_ret(Z_tmp_1, Z_tmp_2);
2185   __ get_method(Z_tos);
2186   __ mem2reg_opt(Z_R1_scratch, Address(Z_tos, Method::const_offset()));
2187   __ load_address(Z_bcp, Address(Z_R1_scratch, Z_tmp_1, ConstMethod::codes_offset()));
2188   __ dispatch_next(vtos, 0 , true);
2189 }
2190 
2191 void TemplateTable::wide_ret() {
2192   transition(vtos, vtos);
2193 
2194   locals_index_wide(Z_tmp_1);
2195   // Get return bci, compute return bcp.
2196   __ mem2reg_opt(Z_tmp_1, aaddress(_masm, Z_tmp_1));
2197   __ profile_ret(Z_tmp_1, Z_tmp_2);
2198   __ get_method(Z_tos);
2199   __ mem2reg_opt(Z_R1_scratch, Address(Z_tos, Method::const_offset()));
2200   __ load_address(Z_bcp, Address(Z_R1_scratch, Z_tmp_1, ConstMethod::codes_offset()));
2201   __ dispatch_next(vtos, 0, true);
2202 }
2203 
2204 void TemplateTable::tableswitch () {
2205   transition(itos, vtos);
2206 
2207   NearLabel default_case, continue_execution;
2208   Register  bcp = Z_ARG5;
2209   // Align bcp.
2210   __ load_address(bcp, at_bcp(BytesPerInt));
2211   __ z_nill(bcp, (-BytesPerInt) & 0xffff);
2212 
2213   // Load lo & hi.
2214   Register low  = Z_tmp_1;
2215   Register high = Z_tmp_2;
2216 
2217   // Load low into 64 bits, since used for address calculation.
2218   __ mem2reg_signed_opt(low, Address(bcp, BytesPerInt));
2219   __ mem2reg_opt(high, Address(bcp, 2 * BytesPerInt), false);
2220   // Sign extend "label" value for address calculation.
2221   __ z_lgfr(Z_tos, Z_tos);
2222 
2223   // Check against lo & hi.
2224   __ compare32_and_branch(Z_tos, low, Assembler::bcondLow, default_case);
2225   __ compare32_and_branch(Z_tos, high, Assembler::bcondHigh, default_case);
2226 
2227   // Lookup dispatch offset.
2228   __ z_sgr(Z_tos, low);
2229   Register jump_table_offset = Z_ARG3;
2230   // Index2offset; index in Z_tos is killed by profile_switch_case.
2231   __ z_sllg(jump_table_offset, Z_tos, LogBytesPerInt);
2232   __ profile_switch_case(Z_tos, Z_ARG4 /*tmp for mdp*/, low/*tmp*/, Z_bytecode/*tmp*/);
2233 
2234   Register index = Z_tmp_2;
2235 
2236   // Load index sign extended for addressing.
2237   __ mem2reg_signed_opt(index, Address(bcp, jump_table_offset, 3 * BytesPerInt));
2238 
2239   // Continue execution.
2240   __ bind(continue_execution);
2241 
2242   // Load next bytecode.
2243   __ z_llgc(Z_bytecode, Address(Z_bcp, index));
2244   __ z_agr(Z_bcp, index); // Advance bcp.
2245   __ dispatch_only(vtos, true);
2246 
2247   // Handle default.
2248   __ bind(default_case);
2249 
2250   __ profile_switch_default(Z_tos);
2251   __ mem2reg_signed_opt(index, Address(bcp));
2252   __ z_bru(continue_execution);
2253 }
2254 
2255 void TemplateTable::lookupswitch () {
2256   transition(itos, itos);
2257   __ stop("lookupswitch bytecode should have been rewritten");
2258 }
2259 
2260 void TemplateTable::fast_linearswitch () {
2261   transition(itos, vtos);
2262 
2263   Label    loop_entry, loop, found, continue_execution;
2264   Register bcp = Z_ARG5;
2265 
2266   // Align bcp.
2267   __ load_address(bcp, at_bcp(BytesPerInt));
2268   __ z_nill(bcp, (-BytesPerInt) & 0xffff);
2269 
2270   // Start search with last case.
2271   Register current_case_offset = Z_tmp_1;
2272 
2273   __ mem2reg_signed_opt(current_case_offset, Address(bcp, BytesPerInt));
2274   __ z_sllg(current_case_offset, current_case_offset, LogBytesPerWord);   // index2bytes
2275   __ z_bru(loop_entry);
2276 
2277   // table search
2278   __ bind(loop);
2279 
2280   __ z_c(Z_tos, Address(bcp, current_case_offset, 2 * BytesPerInt));
2281   __ z_bre(found);
2282 
2283   __ bind(loop_entry);
2284   __ z_aghi(current_case_offset, -2 * BytesPerInt);  // Decrement.
2285   __ z_brnl(loop);
2286 
2287   // default case
2288   Register   offset = Z_tmp_2;
2289 
2290   __ profile_switch_default(Z_tos);
2291   // Load offset sign extended for addressing.
2292   __ mem2reg_signed_opt(offset, Address(bcp));
2293   __ z_bru(continue_execution);
2294 
2295   // Entry found -> get offset.
2296   __ bind(found);
2297   __ mem2reg_signed_opt(offset, Address(bcp, current_case_offset, 3 * BytesPerInt));
2298   // Profile that this case was taken.
2299   Register current_case_idx = Z_ARG4;
2300   __ z_srlg(current_case_idx, current_case_offset, LogBytesPerWord); // bytes2index
2301   __ profile_switch_case(current_case_idx, Z_tos, bcp, Z_bytecode);
2302 
2303   // Continue execution.
2304   __ bind(continue_execution);
2305 
2306   // Load next bytecode.
2307   __ z_llgc(Z_bytecode, Address(Z_bcp, offset, 0));
2308   __ z_agr(Z_bcp, offset); // Advance bcp.
2309   __ dispatch_only(vtos, true);
2310 }
2311 
2312 
2313 void TemplateTable::fast_binaryswitch() {
2314 
2315   transition(itos, vtos);
2316 
2317   // Implementation using the following core algorithm:
2318   //
2319   // int binary_search(int key, LookupswitchPair* array, int n) {
2320   //   // Binary search according to "Methodik des Programmierens" by
2321   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2322   //   int i = 0;
2323   //   int j = n;
2324   //   while (i+1 < j) {
2325   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2326   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2327   //     // where a stands for the array and assuming that the (inexisting)
2328   //     // element a[n] is infinitely big.
2329   //     int h = (i + j) >> 1;
2330   //     // i < h < j
2331   //     if (key < array[h].fast_match()) {
2332   //       j = h;
2333   //     } else {
2334   //       i = h;
2335   //     }
2336   //   }
2337   //   // R: a[i] <= key < a[i+1] or Q
2338   //   // (i.e., if key is within array, i is the correct index)
2339   //   return i;
2340   // }
2341 
2342   // Register allocation
2343   // Note: Since we use the indices in address operands, we do all the
2344   // computation in 64 bits.
2345   const Register key   = Z_tos; // Already set (tosca).
2346   const Register array = Z_tmp_1;
2347   const Register i     = Z_tmp_2;
2348   const Register j     = Z_ARG5;
2349   const Register h     = Z_ARG4;
2350   const Register temp  = Z_R1_scratch;
2351 
2352   // Find array start.
2353   __ load_address(array, at_bcp(3 * BytesPerInt));
2354   __ z_nill(array, (-BytesPerInt) & 0xffff);   // align
2355 
2356   // Initialize i & j.
2357   __ clear_reg(i, true, false);  // i = 0;  Don't set CC.
2358   __ mem2reg_signed_opt(j, Address(array, -BytesPerInt)); // j = length(array);
2359 
2360   // And start.
2361   Label entry;
2362   __ z_bru(entry);
2363 
2364   // binary search loop
2365   {
2366     NearLabel   loop;
2367 
2368     __ bind(loop);
2369 
2370     // int h = (i + j) >> 1;
2371     __ add2reg_with_index(h, 0, i, j); // h = i + j;
2372     __ z_srag(h, h, 1);                // h = (i + j) >> 1;
2373 
2374     // if (key < array[h].fast_match()) {
2375     //   j = h;
2376     // } else {
2377     //   i = h;
2378     // }
2379 
2380     // Convert array[h].match to native byte-ordering before compare.
2381     __ z_sllg(temp, h, LogBytesPerWord);   // index2bytes
2382     __ mem2reg_opt(temp, Address(array, temp), false);
2383 
2384     NearLabel  else_;
2385 
2386     __ compare32_and_branch(key, temp, Assembler::bcondNotLow, else_);
2387     // j = h if (key <  array[h].fast_match())
2388     __ z_lgr(j, h);
2389     __ z_bru(entry); // continue
2390 
2391     __ bind(else_);
2392 
2393     // i = h if (key >= array[h].fast_match())
2394     __ z_lgr(i, h);  // and fallthrough
2395 
2396     // while (i+1 < j)
2397     __ bind(entry);
2398 
2399     // if (i + 1 < j) continue search
2400     __ add2reg(h, 1, i);
2401     __ compare64_and_branch(h, j, Assembler::bcondLow, loop);
2402   }
2403 
2404   // End of binary search, result index is i (must check again!).
2405   NearLabel default_case;
2406 
2407   // h is no longer needed, so use it to hold the byte offset.
2408   __ z_sllg(h, i, LogBytesPerWord);   // index2bytes
2409   __ mem2reg_opt(temp, Address(array, h), false);
2410   __ compare32_and_branch(key, temp, Assembler::bcondNotEqual, default_case);
2411 
2412   // entry found -> j = offset
2413   __ mem2reg_signed_opt(j, Address(array, h, BytesPerInt));
2414   __ profile_switch_case(i, key, array, Z_bytecode);
2415   // Load next bytecode.
2416   __ z_llgc(Z_bytecode, Address(Z_bcp, j));
2417   __ z_agr(Z_bcp, j);       // Advance bcp.
2418   __ dispatch_only(vtos, true);
2419 
2420   // default case -> j = default offset
2421   __ bind(default_case);
2422 
2423   __ profile_switch_default(i);
2424   __ mem2reg_signed_opt(j, Address(array, -2 * BytesPerInt));
2425   // Load next bytecode.
2426   __ z_llgc(Z_bytecode, Address(Z_bcp, j));
2427   __ z_agr(Z_bcp, j);       // Advance bcp.
2428   __ dispatch_only(vtos, true);
2429 }
2430 
2431 void TemplateTable::_return(TosState state) {
2432   transition(state, state);
2433   assert(_desc->calls_vm(),
2434          "inconsistent calls_vm information"); // call in remove_activation
2435 
2436   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2437     Register Rthis  = Z_ARG2;
2438     Register Rklass = Z_ARG5;
2439     Label skip_register_finalizer;
2440     assert(state == vtos, "only valid state");
2441     __ z_lg(Rthis, aaddress(0));
2442     __ load_klass(Rklass, Rthis);
2443     __ testbit(Address(Rklass, Klass::access_flags_offset()), exact_log2(JVM_ACC_HAS_FINALIZER));
2444     __ z_bfalse(skip_register_finalizer);
2445     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), Rthis);
2446     __ bind(skip_register_finalizer);
2447   }
2448 
2449   if (SafepointMechanism::uses_thread_local_poll() && _desc->bytecode() != Bytecodes::_return_register_finalizer) {
2450     Label no_safepoint;
2451     const Address poll_byte_addr(Z_thread, in_bytes(Thread::polling_page_offset()) + 7 /* Big Endian */);
2452     __ z_tm(poll_byte_addr, SafepointMechanism::poll_bit());
2453     __ z_braz(no_safepoint);
2454     __ push(state);
2455     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::at_safepoint));
2456     __ pop(state);
2457     __ bind(no_safepoint);
2458   }
2459 
2460   if (state == itos) {
2461     // Narrow result if state is itos but result type is smaller.
2462     // Need to narrow in the return bytecode rather than in generate_return_entry
2463     // since compiled code callers expect the result to already be narrowed.
2464     __ narrow(Z_tos, Z_tmp_1); /* fall through */
2465   }
2466 
2467   __ remove_activation(state, Z_R14);
2468   __ z_br(Z_R14);
2469 }
2470 
2471 // ----------------------------------------------------------------------------
2472 // NOTE: Cpe_offset is already computed as byte offset, so we must not
2473 // shift it afterwards!
2474 void TemplateTable::resolve_cache_and_index(int byte_no,
2475                                             Register Rcache,
2476                                             Register cpe_offset,
2477                                             size_t index_size) {
2478   BLOCK_COMMENT("resolve_cache_and_index {");
2479   NearLabel      resolved;
2480   const Register bytecode_in_cpcache = Z_R1_scratch;
2481   const int      total_f1_offset = in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset());
2482   assert_different_registers(Rcache, cpe_offset, bytecode_in_cpcache);
2483 
2484   Bytecodes::Code code = bytecode();
2485   switch (code) {
2486     case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2487     case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2488   }
2489 
2490   {
2491     assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2492     __ get_cache_and_index_and_bytecode_at_bcp(Rcache, cpe_offset, bytecode_in_cpcache, byte_no, 1, index_size);
2493     // Have we resolved this bytecode?
2494     __ compare32_and_branch(bytecode_in_cpcache, (int)code, Assembler::bcondEqual, resolved);
2495   }
2496 
2497   // Resolve first time through.
2498   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2499   __ load_const_optimized(Z_ARG2, (int) code);
2500   __ call_VM(noreg, entry, Z_ARG2);
2501 
2502   // Update registers with resolved info.
2503   __ get_cache_and_index_at_bcp(Rcache, cpe_offset, 1, index_size);
2504   __ bind(resolved);
2505   BLOCK_COMMENT("} resolve_cache_and_index");
2506 }
2507 
2508 // The Rcache and index registers must be set before call.
2509 // Index is already a byte offset, don't shift!
2510 void TemplateTable::load_field_cp_cache_entry(Register obj,
2511                                               Register cache,
2512                                               Register index,
2513                                               Register off,
2514                                               Register flags,
2515                                               bool is_static = false) {
2516   assert_different_registers(cache, index, flags, off);
2517   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2518 
2519   // Field offset
2520   __ mem2reg_opt(off, Address(cache, index, cp_base_offset + ConstantPoolCacheEntry::f2_offset()));
2521   // Flags. Must load 64 bits.
2522   __ mem2reg_opt(flags, Address(cache, index, cp_base_offset + ConstantPoolCacheEntry::flags_offset()));
2523 
2524   // klass overwrite register
2525   if (is_static) {
2526     __ mem2reg_opt(obj, Address(cache, index, cp_base_offset + ConstantPoolCacheEntry::f1_offset()));
2527     __ mem2reg_opt(obj, Address(obj, Klass::java_mirror_offset()));
2528     __ resolve_oop_handle(obj);
2529   }
2530 }
2531 
2532 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2533                                                Register method,
2534                                                Register itable_index,
2535                                                Register flags,
2536                                                bool is_invokevirtual,
2537                                                bool is_invokevfinal, // unused
2538                                                bool is_invokedynamic) {
2539   BLOCK_COMMENT("load_invoke_cp_cache_entry {");
2540   // Setup registers.
2541   const Register cache     = Z_ARG1;
2542   const Register cpe_offset= flags;
2543   const ByteSize base_off  = ConstantPoolCache::base_offset();
2544   const ByteSize f1_off    = ConstantPoolCacheEntry::f1_offset();
2545   const ByteSize f2_off    = ConstantPoolCacheEntry::f2_offset();
2546   const ByteSize flags_off = ConstantPoolCacheEntry::flags_offset();
2547   const int method_offset  = in_bytes(base_off + ((byte_no == f2_byte) ? f2_off : f1_off));
2548   const int flags_offset   = in_bytes(base_off + flags_off);
2549   // Access constant pool cache fields.
2550   const int index_offset   = in_bytes(base_off + f2_off);
2551 
2552   assert_different_registers(method, itable_index, flags, cache);
2553   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2554 
2555   if (is_invokevfinal) {
2556     // Already resolved.
2557      assert(itable_index == noreg, "register not used");
2558      __ get_cache_and_index_at_bcp(cache, cpe_offset, 1);
2559   } else {
2560     // Need to resolve.
2561     resolve_cache_and_index(byte_no, cache, cpe_offset, is_invokedynamic ? sizeof(u4) : sizeof(u2));
2562   }
2563   __ z_lg(method, Address(cache, cpe_offset, method_offset));
2564 
2565   if (itable_index != noreg) {
2566     __ z_lg(itable_index, Address(cache, cpe_offset, index_offset));
2567   }
2568 
2569   // Only load the lower 4 bytes and fill high bytes of flags with zeros.
2570   // Callers depend on this zero-extension!!!
2571   // Attention: overwrites cpe_offset == flags
2572   __ z_llgf(flags, Address(cache, cpe_offset, flags_offset + (BytesPerLong-BytesPerInt)));
2573 
2574   BLOCK_COMMENT("} load_invoke_cp_cache_entry");
2575 }
2576 
2577 // The registers cache and index expected to be set before call.
2578 // Correct values of the cache and index registers are preserved.
2579 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2580                                             bool is_static, bool has_tos) {
2581 
2582   // Do the JVMTI work here to avoid disturbing the register state below.
2583   // We use c_rarg registers here because we want to use the register used in
2584   // the call to the VM
2585   if (!JvmtiExport::can_post_field_access()) {
2586     return;
2587   }
2588 
2589   // Check to see if a field access watch has been set before we
2590   // take the time to call into the VM.
2591   Label exit;
2592   assert_different_registers(cache, index, Z_tos);
2593   __ load_absolute_address(Z_tos, (address)JvmtiExport::get_field_access_count_addr());
2594   __ load_and_test_int(Z_R0, Address(Z_tos));
2595   __ z_brz(exit);
2596 
2597   // Index is returned as byte offset, do not shift!
2598   __ get_cache_and_index_at_bcp(Z_ARG3, Z_R1_scratch, 1);
2599 
2600   // cache entry pointer
2601   __ add2reg_with_index(Z_ARG3,
2602                         in_bytes(ConstantPoolCache::base_offset()),
2603                         Z_ARG3, Z_R1_scratch);
2604 
2605   if (is_static) {
2606     __ clear_reg(Z_ARG2, true, false); // NULL object reference. Don't set CC.
2607   } else {
2608     __ mem2reg_opt(Z_ARG2, at_tos());  // Get object pointer without popping it.
2609     __ verify_oop(Z_ARG2);
2610   }
2611   // Z_ARG2: object pointer or NULL
2612   // Z_ARG3: cache entry pointer
2613   __ call_VM(noreg,
2614              CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access),
2615              Z_ARG2, Z_ARG3);
2616   __ get_cache_and_index_at_bcp(cache, index, 1);
2617 
2618   __ bind(exit);
2619 }
2620 
2621 void TemplateTable::pop_and_check_object(Register r) {
2622   __ pop_ptr(r);
2623   __ null_check(r);  // for field access must check obj.
2624   __ verify_oop(r);
2625 }
2626 
2627 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2628   transition(vtos, vtos);
2629 
2630   const Register cache = Z_tmp_1;
2631   const Register index = Z_tmp_2;
2632   const Register obj   = Z_tmp_1;
2633   const Register off   = Z_ARG2;
2634   const Register flags = Z_ARG1;
2635   const Register bc    = Z_tmp_1;  // Uses same reg as obj, so don't mix them.
2636 
2637   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2638   jvmti_post_field_access(cache, index, is_static, false);
2639   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2640 
2641   if (!is_static) {
2642     // Obj is on the stack.
2643     pop_and_check_object(obj);
2644   }
2645 
2646   // Displacement is 0, so any store instruction will be fine on any CPU.
2647   const Address field(obj, off);
2648 
2649   Label    is_Byte, is_Bool, is_Int, is_Short, is_Char,
2650            is_Long, is_Float, is_Object, is_Double;
2651   Label    is_badState8, is_badState9, is_badStateA, is_badStateB,
2652            is_badStateC, is_badStateD, is_badStateE, is_badStateF,
2653            is_badState;
2654   Label    branchTable, atosHandler,  Done;
2655   Register br_tab       = Z_R1_scratch;
2656   bool     do_rewrite   = !is_static && (rc == may_rewrite);
2657   bool     dont_rewrite = (is_static || (rc == may_not_rewrite));
2658 
2659   assert(do_rewrite == !dont_rewrite, "Oops, code is not fit for that");
2660   assert(btos == 0, "change code, btos != 0");
2661 
2662   // Calculate branch table size. Generated code size depends on ASSERT and on bytecode rewriting.
2663 #ifdef ASSERT
2664   const unsigned int bsize = dont_rewrite ? BTB_MINSIZE*1 : BTB_MINSIZE*4;
2665 #else
2666   const unsigned int bsize = dont_rewrite ? BTB_MINSIZE*1 : BTB_MINSIZE*4;
2667 #endif
2668 
2669   // Calculate address of branch table entry and branch there.
2670   {
2671     const int bit_shift = exact_log2(bsize); // Size of each branch table entry.
2672     const int r_bitpos  = 63 - bit_shift;
2673     const int l_bitpos  = r_bitpos - ConstantPoolCacheEntry::tos_state_bits + 1;
2674     const int n_rotate  = (bit_shift-ConstantPoolCacheEntry::tos_state_shift);
2675     __ z_larl(br_tab, branchTable);
2676     __ rotate_then_insert(flags, flags, l_bitpos, r_bitpos, n_rotate, true);
2677   }
2678   __ z_bc(Assembler::bcondAlways, 0, flags, br_tab);
2679 
2680   __ align_address(bsize);
2681   BIND(branchTable);
2682 
2683   // btos
2684   BTB_BEGIN(is_Byte, bsize, "getfield_or_static:is_Byte");
2685   __ z_lb(Z_tos, field);
2686   __ push(btos);
2687   // Rewrite bytecode to be faster.
2688   if (do_rewrite) {
2689     patch_bytecode(Bytecodes::_fast_bgetfield, bc, Z_ARG5);
2690   }
2691   __ z_bru(Done);
2692   BTB_END(is_Byte, bsize, "getfield_or_static:is_Byte");
2693 
2694   // ztos
2695   BTB_BEGIN(is_Bool, bsize, "getfield_or_static:is_Bool");
2696   __ z_lb(Z_tos, field);
2697   __ push(ztos);
2698   // Rewrite bytecode to be faster.
2699   if (do_rewrite) {
2700     // Use btos rewriting, no truncating to t/f bit is needed for getfield.
2701     patch_bytecode(Bytecodes::_fast_bgetfield, bc, Z_ARG5);
2702   }
2703   __ z_bru(Done);
2704   BTB_END(is_Bool, bsize, "getfield_or_static:is_Bool");
2705 
2706   // ctos
2707   BTB_BEGIN(is_Char, bsize, "getfield_or_static:is_Char");
2708   // Load into 64 bits, works on all CPUs.
2709   __ z_llgh(Z_tos, field);
2710   __ push(ctos);
2711   // Rewrite bytecode to be faster.
2712   if (do_rewrite) {
2713     patch_bytecode(Bytecodes::_fast_cgetfield, bc, Z_ARG5);
2714   }
2715   __ z_bru(Done);
2716   BTB_END(is_Char, bsize, "getfield_or_static:is_Char");
2717 
2718   // stos
2719   BTB_BEGIN(is_Short, bsize, "getfield_or_static:is_Short");
2720   __ z_lh(Z_tos, field);
2721   __ push(stos);
2722   // Rewrite bytecode to be faster.
2723   if (do_rewrite) {
2724     patch_bytecode(Bytecodes::_fast_sgetfield, bc, Z_ARG5);
2725   }
2726   __ z_bru(Done);
2727   BTB_END(is_Short, bsize, "getfield_or_static:is_Short");
2728 
2729   // itos
2730   BTB_BEGIN(is_Int, bsize, "getfield_or_static:is_Int");
2731   __ mem2reg_opt(Z_tos, field, false);
2732   __ push(itos);
2733   // Rewrite bytecode to be faster.
2734   if (do_rewrite) {
2735     patch_bytecode(Bytecodes::_fast_igetfield, bc, Z_ARG5);
2736   }
2737   __ z_bru(Done);
2738   BTB_END(is_Int, bsize, "getfield_or_static:is_Int");
2739 
2740   // ltos
2741   BTB_BEGIN(is_Long, bsize, "getfield_or_static:is_Long");
2742   __ mem2reg_opt(Z_tos, field);
2743   __ push(ltos);
2744   // Rewrite bytecode to be faster.
2745   if (do_rewrite) {
2746     patch_bytecode(Bytecodes::_fast_lgetfield, bc, Z_ARG5);
2747   }
2748   __ z_bru(Done);
2749   BTB_END(is_Long, bsize, "getfield_or_static:is_Long");
2750 
2751   // ftos
2752   BTB_BEGIN(is_Float, bsize, "getfield_or_static:is_Float");
2753   __ mem2freg_opt(Z_ftos, field, false);
2754   __ push(ftos);
2755   // Rewrite bytecode to be faster.
2756   if (do_rewrite) {
2757     patch_bytecode(Bytecodes::_fast_fgetfield, bc, Z_ARG5);
2758   }
2759   __ z_bru(Done);
2760   BTB_END(is_Float, bsize, "getfield_or_static:is_Float");
2761 
2762   // dtos
2763   BTB_BEGIN(is_Double, bsize, "getfield_or_static:is_Double");
2764   __ mem2freg_opt(Z_ftos, field);
2765   __ push(dtos);
2766   // Rewrite bytecode to be faster.
2767   if (do_rewrite) {
2768     patch_bytecode(Bytecodes::_fast_dgetfield, bc, Z_ARG5);
2769   }
2770   __ z_bru(Done);
2771   BTB_END(is_Double, bsize, "getfield_or_static:is_Double");
2772 
2773   // atos
2774   BTB_BEGIN(is_Object, bsize, "getfield_or_static:is_Object");
2775   __ z_bru(atosHandler);
2776   BTB_END(is_Object, bsize, "getfield_or_static:is_Object");
2777 
2778   // Bad state detection comes at no extra runtime cost.
2779   BTB_BEGIN(is_badState8, bsize, "getfield_or_static:is_badState8");
2780   __ z_illtrap();
2781   __ z_bru(is_badState);
2782   BTB_END( is_badState8, bsize, "getfield_or_static:is_badState8");
2783   BTB_BEGIN(is_badState9, bsize, "getfield_or_static:is_badState9");
2784   __ z_illtrap();
2785   __ z_bru(is_badState);
2786   BTB_END( is_badState9, bsize, "getfield_or_static:is_badState9");
2787   BTB_BEGIN(is_badStateA, bsize, "getfield_or_static:is_badStateA");
2788   __ z_illtrap();
2789   __ z_bru(is_badState);
2790   BTB_END( is_badStateA, bsize, "getfield_or_static:is_badStateA");
2791   BTB_BEGIN(is_badStateB, bsize, "getfield_or_static:is_badStateB");
2792   __ z_illtrap();
2793   __ z_bru(is_badState);
2794   BTB_END( is_badStateB, bsize, "getfield_or_static:is_badStateB");
2795   BTB_BEGIN(is_badStateC, bsize, "getfield_or_static:is_badStateC");
2796   __ z_illtrap();
2797   __ z_bru(is_badState);
2798   BTB_END( is_badStateC, bsize, "getfield_or_static:is_badStateC");
2799   BTB_BEGIN(is_badStateD, bsize, "getfield_or_static:is_badStateD");
2800   __ z_illtrap();
2801   __ z_bru(is_badState);
2802   BTB_END( is_badStateD, bsize, "getfield_or_static:is_badStateD");
2803   BTB_BEGIN(is_badStateE, bsize, "getfield_or_static:is_badStateE");
2804   __ z_illtrap();
2805   __ z_bru(is_badState);
2806   BTB_END( is_badStateE, bsize, "getfield_or_static:is_badStateE");
2807   BTB_BEGIN(is_badStateF, bsize, "getfield_or_static:is_badStateF");
2808   __ z_illtrap();
2809   __ z_bru(is_badState);
2810   BTB_END( is_badStateF, bsize, "getfield_or_static:is_badStateF");
2811 
2812   __ align_address(64);
2813   BIND(is_badState);  // Do this outside branch table. Needs a lot of space.
2814   {
2815     unsigned int b_off = __ offset();
2816     if (is_static) {
2817       __ stop_static("Bad state in getstatic");
2818     } else {
2819       __ stop_static("Bad state in getfield");
2820     }
2821     unsigned int e_off = __ offset();
2822   }
2823 
2824   __ align_address(64);
2825   BIND(atosHandler);  // Oops are really complicated to handle.
2826                       // There is a lot of code generated.
2827                       // Therefore: generate the handler outside of branch table.
2828                       // There is no performance penalty. The additional branch
2829                       // to here is compensated for by the fallthru to "Done".
2830   {
2831     unsigned int b_off = __ offset();
2832     __ load_heap_oop(Z_tos, field);
2833     __ verify_oop(Z_tos);
2834     __ push(atos);
2835     if (do_rewrite) {
2836       patch_bytecode(Bytecodes::_fast_agetfield, bc, Z_ARG5);
2837     }
2838     unsigned int e_off = __ offset();
2839   }
2840 
2841   BIND(Done);
2842 }
2843 
2844 void TemplateTable::getfield(int byte_no) {
2845   BLOCK_COMMENT("getfield  {");
2846   getfield_or_static(byte_no, false);
2847   BLOCK_COMMENT("} getfield");
2848 }
2849 
2850 void TemplateTable::nofast_getfield(int byte_no) {
2851   getfield_or_static(byte_no, false, may_not_rewrite);
2852 }
2853 
2854 void TemplateTable::getstatic(int byte_no) {
2855   BLOCK_COMMENT("getstatic {");
2856   getfield_or_static(byte_no, true);
2857   BLOCK_COMMENT("} getstatic");
2858 }
2859 
2860 // The registers cache and index expected to be set before call.  The
2861 // function may destroy various registers, just not the cache and
2862 // index registers.
2863 void TemplateTable::jvmti_post_field_mod(Register cache,
2864                                          Register index, bool is_static) {
2865   transition(vtos, vtos);
2866 
2867   if (!JvmtiExport::can_post_field_modification()) {
2868     return;
2869   }
2870 
2871   BLOCK_COMMENT("jvmti_post_field_mod {");
2872 
2873   // Check to see if a field modification watch has been set before
2874   // we take the time to call into the VM.
2875   Label    L1;
2876   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2877   assert_different_registers(cache, index, Z_tos);
2878 
2879   __ load_absolute_address(Z_tos, (address)JvmtiExport::get_field_modification_count_addr());
2880   __ load_and_test_int(Z_R0, Address(Z_tos));
2881   __ z_brz(L1);
2882 
2883   // Index is returned as byte offset, do not shift!
2884   __ get_cache_and_index_at_bcp(Z_ARG3, Z_R1_scratch, 1);
2885 
2886   if (is_static) {
2887     // Life is simple. Null out the object pointer.
2888     __ clear_reg(Z_ARG2, true, false);   // Don't set CC.
2889   } else {
2890     // Life is harder. The stack holds the value on top, followed by
2891     // the object. We don't know the size of the value, though. It
2892     // could be one or two words depending on its type. As a result,
2893     // we must find the type to determine where the object is.
2894     __ mem2reg_opt(Z_ARG4,
2895                    Address(Z_ARG3, Z_R1_scratch,
2896                            in_bytes(cp_base_offset + ConstantPoolCacheEntry::flags_offset()) +
2897                            (BytesPerLong - BytesPerInt)),
2898                    false);
2899     __ z_srl(Z_ARG4, ConstantPoolCacheEntry::tos_state_shift);
2900     // Make sure we don't need to mask Z_ARG4 for tos_state after the above shift.
2901     ConstantPoolCacheEntry::verify_tos_state_shift();
2902     __ mem2reg_opt(Z_ARG2, at_tos(1));  // Initially assume a one word jvalue.
2903 
2904     NearLabel   load_dtos, cont;
2905 
2906     __ compareU32_and_branch(Z_ARG4, (intptr_t) ltos,
2907                               Assembler::bcondNotEqual, load_dtos);
2908     __ mem2reg_opt(Z_ARG2, at_tos(2)); // ltos (two word jvalue)
2909     __ z_bru(cont);
2910 
2911     __ bind(load_dtos);
2912     __ compareU32_and_branch(Z_ARG4, (intptr_t)dtos, Assembler::bcondNotEqual, cont);
2913     __ mem2reg_opt(Z_ARG2, at_tos(2)); // dtos (two word jvalue)
2914 
2915     __ bind(cont);
2916   }
2917   // cache entry pointer
2918 
2919   __ add2reg_with_index(Z_ARG3, in_bytes(cp_base_offset), Z_ARG3, Z_R1_scratch);
2920 
2921   // object(tos)
2922   __ load_address(Z_ARG4, Address(Z_esp, Interpreter::stackElementSize));
2923   // Z_ARG2: object pointer set up above (NULL if static)
2924   // Z_ARG3: cache entry pointer
2925   // Z_ARG4: jvalue object on the stack
2926   __ call_VM(noreg,
2927              CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification),
2928              Z_ARG2, Z_ARG3, Z_ARG4);
2929   __ get_cache_and_index_at_bcp(cache, index, 1);
2930 
2931   __ bind(L1);
2932   BLOCK_COMMENT("} jvmti_post_field_mod");
2933 }
2934 
2935 
2936 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2937   transition(vtos, vtos);
2938 
2939   const Register cache         = Z_tmp_1;
2940   const Register index         = Z_ARG5;
2941   const Register obj           = Z_tmp_1;
2942   const Register off           = Z_tmp_2;
2943   const Register flags         = Z_R1_scratch;
2944   const Register br_tab        = Z_ARG5;
2945   const Register bc            = Z_tmp_1;
2946   const Register oopStore_tmp1 = Z_R1_scratch;
2947   const Register oopStore_tmp2 = Z_ARG5;
2948   const Register oopStore_tmp3 = Z_R0_scratch;
2949 
2950   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2951   jvmti_post_field_mod(cache, index, is_static);
2952   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2953   // begin of life for:
2954   //   obj, off   long life range
2955   //   flags      short life range, up to branch into branch table
2956   // end of life for:
2957   //   cache, index
2958 
2959   const Address field(obj, off);
2960   Label is_Byte, is_Bool, is_Int, is_Short, is_Char,
2961         is_Long, is_Float, is_Object, is_Double;
2962   Label is_badState8, is_badState9, is_badStateA, is_badStateB,
2963         is_badStateC, is_badStateD, is_badStateE, is_badStateF,
2964         is_badState;
2965   Label branchTable, atosHandler, Done;
2966   bool  do_rewrite   = !is_static && (rc == may_rewrite);
2967   bool  dont_rewrite = (is_static || (rc == may_not_rewrite));
2968 
2969   assert(do_rewrite == !dont_rewrite, "Oops, code is not fit for that");
2970 
2971   assert(btos == 0, "change code, btos != 0");
2972 
2973 #ifdef ASSERT
2974   const unsigned int bsize = is_static ? BTB_MINSIZE*1 : BTB_MINSIZE*4;
2975 #else
2976   const unsigned int bsize = is_static ? BTB_MINSIZE*1 : BTB_MINSIZE*8;
2977 #endif
2978 
2979   // Calculate address of branch table entry and branch there.
2980   {
2981     const int bit_shift = exact_log2(bsize); // Size of each branch table entry.
2982     const int r_bitpos  = 63 - bit_shift;
2983     const int l_bitpos  = r_bitpos - ConstantPoolCacheEntry::tos_state_bits + 1;
2984     const int n_rotate  = (bit_shift-ConstantPoolCacheEntry::tos_state_shift);
2985     __ z_larl(br_tab, branchTable);
2986     __ rotate_then_insert(flags, flags, l_bitpos, r_bitpos, n_rotate, true);
2987     __ z_bc(Assembler::bcondAlways, 0, flags, br_tab);
2988   }
2989   // end of life for:
2990   //   flags, br_tab
2991 
2992   __ align_address(bsize);
2993   BIND(branchTable);
2994 
2995   // btos
2996   BTB_BEGIN(is_Byte, bsize, "putfield_or_static:is_Byte");
2997   __ pop(btos);
2998   if (!is_static) {
2999     pop_and_check_object(obj);
3000   }
3001   __ z_stc(Z_tos, field);
3002   if (do_rewrite) {
3003     patch_bytecode(Bytecodes::_fast_bputfield, bc, Z_ARG5, true, byte_no);
3004   }
3005   __ z_bru(Done);
3006   BTB_END( is_Byte, bsize, "putfield_or_static:is_Byte");
3007 
3008   // ztos
3009   BTB_BEGIN(is_Bool, bsize, "putfield_or_static:is_Bool");
3010   __ pop(ztos);
3011   if (!is_static) {
3012     pop_and_check_object(obj);
3013   }
3014   __ z_nilf(Z_tos, 0x1);
3015   __ z_stc(Z_tos, field);
3016   if (do_rewrite) {
3017     patch_bytecode(Bytecodes::_fast_zputfield, bc, Z_ARG5, true, byte_no);
3018   }
3019   __ z_bru(Done);
3020   BTB_END(is_Bool, bsize, "putfield_or_static:is_Bool");
3021 
3022   // ctos
3023   BTB_BEGIN(is_Char, bsize, "putfield_or_static:is_Char");
3024   __ pop(ctos);
3025   if (!is_static) {
3026     pop_and_check_object(obj);
3027   }
3028   __ z_sth(Z_tos, field);
3029   if (do_rewrite) {
3030     patch_bytecode(Bytecodes::_fast_cputfield, bc, Z_ARG5, true, byte_no);
3031   }
3032   __ z_bru(Done);
3033   BTB_END( is_Char, bsize, "putfield_or_static:is_Char");
3034 
3035   // stos
3036   BTB_BEGIN(is_Short, bsize, "putfield_or_static:is_Short");
3037   __ pop(stos);
3038   if (!is_static) {
3039     pop_and_check_object(obj);
3040   }
3041   __ z_sth(Z_tos, field);
3042   if (do_rewrite) {
3043     patch_bytecode(Bytecodes::_fast_sputfield, bc, Z_ARG5, true, byte_no);
3044   }
3045   __ z_bru(Done);
3046   BTB_END( is_Short, bsize, "putfield_or_static:is_Short");
3047 
3048   // itos
3049   BTB_BEGIN(is_Int, bsize, "putfield_or_static:is_Int");
3050   __ pop(itos);
3051   if (!is_static) {
3052     pop_and_check_object(obj);
3053   }
3054   __ reg2mem_opt(Z_tos, field, false);
3055   if (do_rewrite) {
3056     patch_bytecode(Bytecodes::_fast_iputfield, bc, Z_ARG5, true, byte_no);
3057   }
3058   __ z_bru(Done);
3059   BTB_END( is_Int, bsize, "putfield_or_static:is_Int");
3060 
3061   // ltos
3062   BTB_BEGIN(is_Long, bsize, "putfield_or_static:is_Long");
3063   __ pop(ltos);
3064   if (!is_static) {
3065     pop_and_check_object(obj);
3066   }
3067   __ reg2mem_opt(Z_tos, field);
3068   if (do_rewrite) {
3069     patch_bytecode(Bytecodes::_fast_lputfield, bc, Z_ARG5, true, byte_no);
3070   }
3071   __ z_bru(Done);
3072   BTB_END( is_Long, bsize, "putfield_or_static:is_Long");
3073 
3074   // ftos
3075   BTB_BEGIN(is_Float, bsize, "putfield_or_static:is_Float");
3076   __ pop(ftos);
3077   if (!is_static) {
3078     pop_and_check_object(obj);
3079   }
3080   __ freg2mem_opt(Z_ftos, field, false);
3081   if (do_rewrite) {
3082     patch_bytecode(Bytecodes::_fast_fputfield, bc, Z_ARG5, true, byte_no);
3083   }
3084   __ z_bru(Done);
3085   BTB_END( is_Float, bsize, "putfield_or_static:is_Float");
3086 
3087   // dtos
3088   BTB_BEGIN(is_Double, bsize, "putfield_or_static:is_Double");
3089   __ pop(dtos);
3090   if (!is_static) {
3091     pop_and_check_object(obj);
3092   }
3093   __ freg2mem_opt(Z_ftos, field);
3094   if (do_rewrite) {
3095     patch_bytecode(Bytecodes::_fast_dputfield, bc, Z_ARG5, true, byte_no);
3096   }
3097   __ z_bru(Done);
3098   BTB_END( is_Double, bsize, "putfield_or_static:is_Double");
3099 
3100   // atos
3101   BTB_BEGIN(is_Object, bsize, "putfield_or_static:is_Object");
3102   __ z_bru(atosHandler);
3103   BTB_END( is_Object, bsize, "putfield_or_static:is_Object");
3104 
3105   // Bad state detection comes at no extra runtime cost.
3106   BTB_BEGIN(is_badState8, bsize, "putfield_or_static:is_badState8");
3107   __ z_illtrap();
3108   __ z_bru(is_badState);
3109   BTB_END( is_badState8, bsize, "putfield_or_static:is_badState8");
3110   BTB_BEGIN(is_badState9, bsize, "putfield_or_static:is_badState9");
3111   __ z_illtrap();
3112   __ z_bru(is_badState);
3113   BTB_END( is_badState9, bsize, "putfield_or_static:is_badState9");
3114   BTB_BEGIN(is_badStateA, bsize, "putfield_or_static:is_badStateA");
3115   __ z_illtrap();
3116   __ z_bru(is_badState);
3117   BTB_END( is_badStateA, bsize, "putfield_or_static:is_badStateA");
3118   BTB_BEGIN(is_badStateB, bsize, "putfield_or_static:is_badStateB");
3119   __ z_illtrap();
3120   __ z_bru(is_badState);
3121   BTB_END( is_badStateB, bsize, "putfield_or_static:is_badStateB");
3122   BTB_BEGIN(is_badStateC, bsize, "putfield_or_static:is_badStateC");
3123   __ z_illtrap();
3124   __ z_bru(is_badState);
3125   BTB_END( is_badStateC, bsize, "putfield_or_static:is_badStateC");
3126   BTB_BEGIN(is_badStateD, bsize, "putfield_or_static:is_badStateD");
3127   __ z_illtrap();
3128   __ z_bru(is_badState);
3129   BTB_END( is_badStateD, bsize, "putfield_or_static:is_badStateD");
3130   BTB_BEGIN(is_badStateE, bsize, "putfield_or_static:is_badStateE");
3131   __ z_illtrap();
3132   __ z_bru(is_badState);
3133   BTB_END( is_badStateE, bsize, "putfield_or_static:is_badStateE");
3134   BTB_BEGIN(is_badStateF, bsize, "putfield_or_static:is_badStateF");
3135   __ z_illtrap();
3136   __ z_bru(is_badState);
3137   BTB_END( is_badStateF, bsize, "putfield_or_static:is_badStateF");
3138 
3139   __ align_address(64);
3140   BIND(is_badState);  // Do this outside branch table. Needs a lot of space.
3141   {
3142     unsigned int b_off = __ offset();
3143     if (is_static) __ stop_static("Bad state in putstatic");
3144     else            __ stop_static("Bad state in putfield");
3145     unsigned int e_off = __ offset();
3146   }
3147 
3148   __ align_address(64);
3149   BIND(atosHandler);  // Oops are really complicated to handle.
3150                       // There is a lot of code generated.
3151                       // Therefore: generate the handler outside of branch table.
3152                       // There is no performance penalty. The additional branch
3153                       // to here is compensated for by the fallthru to "Done".
3154   {
3155     unsigned int b_off = __ offset();
3156     __ pop(atos);
3157     if (!is_static) {
3158       pop_and_check_object(obj);
3159     }
3160     // Store into the field
3161     do_oop_store(_masm, obj, off, Z_tos, false,
3162                  oopStore_tmp1, oopStore_tmp2, oopStore_tmp3, _bs->kind(), false);
3163     if (do_rewrite) {
3164       patch_bytecode(Bytecodes::_fast_aputfield, bc, Z_ARG5, true, byte_no);
3165     }
3166     // __ z_bru(Done); // fallthru
3167     unsigned int e_off = __ offset();
3168   }
3169 
3170   BIND(Done);
3171 
3172   // Check for volatile store.
3173   Label notVolatile;
3174 
3175   __ testbit(Z_ARG4, ConstantPoolCacheEntry::is_volatile_shift);
3176   __ z_brz(notVolatile);
3177   __ z_fence();
3178 
3179   BIND(notVolatile);
3180 }
3181 
3182 void TemplateTable::putfield(int byte_no) {
3183   BLOCK_COMMENT("putfield  {");
3184   putfield_or_static(byte_no, false);
3185   BLOCK_COMMENT("} putfield");
3186 }
3187 
3188 void TemplateTable::nofast_putfield(int byte_no) {
3189   putfield_or_static(byte_no, false, may_not_rewrite);
3190 }
3191 
3192 void TemplateTable::putstatic(int byte_no) {
3193   BLOCK_COMMENT("putstatic {");
3194   putfield_or_static(byte_no, true);
3195   BLOCK_COMMENT("} putstatic");
3196 }
3197 
3198 // Push the tos value back to the stack.
3199 // gc will find oops there and update.
3200 void TemplateTable::jvmti_post_fast_field_mod() {
3201 
3202   if (!JvmtiExport::can_post_field_modification()) {
3203     return;
3204   }
3205 
3206   // Check to see if a field modification watch has been set before
3207   // we take the time to call into the VM.
3208   Label   exit;
3209 
3210   BLOCK_COMMENT("jvmti_post_fast_field_mod {");
3211 
3212   __ load_absolute_address(Z_R1_scratch,
3213                            (address) JvmtiExport::get_field_modification_count_addr());
3214   __ load_and_test_int(Z_R0_scratch, Address(Z_R1_scratch));
3215   __ z_brz(exit);
3216 
3217   Register obj = Z_tmp_1;
3218 
3219   __ pop_ptr(obj);                  // Copy the object pointer from tos.
3220   __ verify_oop(obj);
3221   __ push_ptr(obj);                 // Put the object pointer back on tos.
3222 
3223   // Save tos values before call_VM() clobbers them. Since we have
3224   // to do it for every data type, we use the saved values as the
3225   // jvalue object.
3226   switch (bytecode()) {          // Load values into the jvalue object.
3227     case Bytecodes::_fast_aputfield:
3228       __ push_ptr(Z_tos);
3229       break;
3230     case Bytecodes::_fast_bputfield:
3231     case Bytecodes::_fast_zputfield:
3232     case Bytecodes::_fast_sputfield:
3233     case Bytecodes::_fast_cputfield:
3234     case Bytecodes::_fast_iputfield:
3235       __ push_i(Z_tos);
3236       break;
3237     case Bytecodes::_fast_dputfield:
3238       __ push_d();
3239       break;
3240     case Bytecodes::_fast_fputfield:
3241       __ push_f();
3242       break;
3243     case Bytecodes::_fast_lputfield:
3244       __ push_l(Z_tos);
3245       break;
3246 
3247     default:
3248       ShouldNotReachHere();
3249   }
3250 
3251   // jvalue on the stack
3252   __ load_address(Z_ARG4, Address(Z_esp, Interpreter::stackElementSize));
3253   // Access constant pool cache entry.
3254   __ get_cache_entry_pointer_at_bcp(Z_ARG3, Z_tos, 1);
3255   __ verify_oop(obj);
3256 
3257   // obj   : object pointer copied above
3258   // Z_ARG3: cache entry pointer
3259   // Z_ARG4: jvalue object on the stack
3260   __ call_VM(noreg,
3261              CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification),
3262              obj, Z_ARG3, Z_ARG4);
3263 
3264   switch (bytecode()) {             // Restore tos values.
3265     case Bytecodes::_fast_aputfield:
3266       __ pop_ptr(Z_tos);
3267       break;
3268     case Bytecodes::_fast_bputfield:
3269     case Bytecodes::_fast_zputfield:
3270     case Bytecodes::_fast_sputfield:
3271     case Bytecodes::_fast_cputfield:
3272     case Bytecodes::_fast_iputfield:
3273       __ pop_i(Z_tos);
3274       break;
3275     case Bytecodes::_fast_dputfield:
3276       __ pop_d(Z_ftos);
3277       break;
3278     case Bytecodes::_fast_fputfield:
3279       __ pop_f(Z_ftos);
3280       break;
3281     case Bytecodes::_fast_lputfield:
3282       __ pop_l(Z_tos);
3283       break;
3284   }
3285 
3286   __ bind(exit);
3287   BLOCK_COMMENT("} jvmti_post_fast_field_mod");
3288 }
3289 
3290 void TemplateTable::fast_storefield(TosState state) {
3291   transition(state, vtos);
3292 
3293   ByteSize base = ConstantPoolCache::base_offset();
3294   jvmti_post_fast_field_mod();
3295 
3296   // Access constant pool cache.
3297   Register cache = Z_tmp_1;
3298   Register index = Z_tmp_2;
3299   Register flags = Z_ARG5;
3300 
3301   // Index comes in bytes, don't shift afterwards!
3302   __ get_cache_and_index_at_bcp(cache, index, 1);
3303 
3304   // Test for volatile.
3305   assert(!flags->is_volatile(), "do_oop_store could perform leaf RT call");
3306   __ z_lg(flags, Address(cache, index, base + ConstantPoolCacheEntry::flags_offset()));
3307 
3308   // Replace index with field offset from cache entry.
3309   Register field_offset = index;
3310   __ z_lg(field_offset, Address(cache, index, base + ConstantPoolCacheEntry::f2_offset()));
3311 
3312   // Get object from stack.
3313   Register   obj = cache;
3314 
3315   pop_and_check_object(obj);
3316 
3317   // field address
3318   const Address   field(obj, field_offset);
3319 
3320   // access field
3321   switch (bytecode()) {
3322     case Bytecodes::_fast_aputfield:
3323       do_oop_store(_masm, obj, field_offset, Z_tos, false,
3324                    Z_ARG2, Z_ARG3, Z_ARG4, _bs->kind(), false);
3325       break;
3326     case Bytecodes::_fast_lputfield:
3327       __ reg2mem_opt(Z_tos, field);
3328       break;
3329     case Bytecodes::_fast_iputfield:
3330       __ reg2mem_opt(Z_tos, field, false);
3331       break;
3332     case Bytecodes::_fast_zputfield:
3333       __ z_nilf(Z_tos, 0x1);
3334       // fall through to bputfield
3335     case Bytecodes::_fast_bputfield:
3336       __ z_stc(Z_tos, field);
3337       break;
3338     case Bytecodes::_fast_sputfield:
3339       // fall through
3340     case Bytecodes::_fast_cputfield:
3341       __ z_sth(Z_tos, field);
3342       break;
3343     case Bytecodes::_fast_fputfield:
3344       __ freg2mem_opt(Z_ftos, field, false);
3345       break;
3346     case Bytecodes::_fast_dputfield:
3347       __ freg2mem_opt(Z_ftos, field);
3348       break;
3349     default:
3350       ShouldNotReachHere();
3351   }
3352 
3353   //  Check for volatile store.
3354   Label notVolatile;
3355 
3356   __ testbit(flags, ConstantPoolCacheEntry::is_volatile_shift);
3357   __ z_brz(notVolatile);
3358   __ z_fence();
3359 
3360   __ bind(notVolatile);
3361 }
3362 
3363 void TemplateTable::fast_accessfield(TosState state) {
3364   transition(atos, state);
3365 
3366   Register obj = Z_tos;
3367 
3368   // Do the JVMTI work here to avoid disturbing the register state below
3369   if (JvmtiExport::can_post_field_access()) {
3370     // Check to see if a field access watch has been set before we
3371     // take the time to call into the VM.
3372     Label cont;
3373 
3374     __ load_absolute_address(Z_R1_scratch,
3375                              (address)JvmtiExport::get_field_access_count_addr());
3376     __ load_and_test_int(Z_R0_scratch, Address(Z_R1_scratch));
3377     __ z_brz(cont);
3378 
3379     // Access constant pool cache entry.
3380 
3381     __ get_cache_entry_pointer_at_bcp(Z_ARG3, Z_tmp_1, 1);
3382     __ verify_oop(obj);
3383     __ push_ptr(obj);  // Save object pointer before call_VM() clobbers it.
3384     __ z_lgr(Z_ARG2, obj);
3385 
3386     // Z_ARG2: object pointer copied above
3387     // Z_ARG3: cache entry pointer
3388     __ call_VM(noreg,
3389                CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access),
3390                Z_ARG2, Z_ARG3);
3391     __ pop_ptr(obj); // Restore object pointer.
3392 
3393     __ bind(cont);
3394   }
3395 
3396   // Access constant pool cache.
3397   Register   cache = Z_tmp_1;
3398   Register   index = Z_tmp_2;
3399 
3400   // Index comes in bytes, don't shift afterwards!
3401   __ get_cache_and_index_at_bcp(cache, index, 1);
3402   // Replace index with field offset from cache entry.
3403   __ mem2reg_opt(index,
3404                  Address(cache, index,
3405                          ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset()));
3406 
3407   __ verify_oop(obj);
3408   __ null_check(obj);
3409 
3410   Address field(obj, index);
3411 
3412   // access field
3413   switch (bytecode()) {
3414     case Bytecodes::_fast_agetfield:
3415       __ load_heap_oop(Z_tos, field);
3416       __ verify_oop(Z_tos);
3417       return;
3418     case Bytecodes::_fast_lgetfield:
3419       __ mem2reg_opt(Z_tos, field);
3420       return;
3421     case Bytecodes::_fast_igetfield:
3422       __ mem2reg_opt(Z_tos, field, false);
3423       return;
3424     case Bytecodes::_fast_bgetfield:
3425       __ z_lb(Z_tos, field);
3426       return;
3427     case Bytecodes::_fast_sgetfield:
3428       __ z_lh(Z_tos, field);
3429       return;
3430     case Bytecodes::_fast_cgetfield:
3431       __ z_llgh(Z_tos, field);   // Load into 64 bits, works on all CPUs.
3432       return;
3433     case Bytecodes::_fast_fgetfield:
3434       __ mem2freg_opt(Z_ftos, field, false);
3435       return;
3436     case Bytecodes::_fast_dgetfield:
3437       __ mem2freg_opt(Z_ftos, field);
3438       return;
3439     default:
3440       ShouldNotReachHere();
3441   }
3442 }
3443 
3444 void TemplateTable::fast_xaccess(TosState state) {
3445   transition(vtos, state);
3446 
3447   Register receiver = Z_tos;
3448   // Get receiver.
3449   __ mem2reg_opt(Z_tos, aaddress(0));
3450 
3451   // Access constant pool cache.
3452   Register cache = Z_tmp_1;
3453   Register index = Z_tmp_2;
3454 
3455   // Index comes in bytes, don't shift afterwards!
3456   __ get_cache_and_index_at_bcp(cache, index, 2);
3457   // Replace index with field offset from cache entry.
3458   __ mem2reg_opt(index,
3459                  Address(cache, index,
3460                          ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset()));
3461 
3462   // Make sure exception is reported in correct bcp range (getfield is
3463   // next instruction).
3464   __ add2reg(Z_bcp, 1);
3465   __ null_check(receiver);
3466   switch (state) {
3467     case itos:
3468       __ mem2reg_opt(Z_tos, Address(receiver, index), false);
3469       break;
3470     case atos:
3471       __ load_heap_oop(Z_tos, Address(receiver, index));
3472       __ verify_oop(Z_tos);
3473       break;
3474     case ftos:
3475       __ mem2freg_opt(Z_ftos, Address(receiver, index));
3476       break;
3477     default:
3478       ShouldNotReachHere();
3479   }
3480 
3481   // Reset bcp to original position.
3482   __ add2reg(Z_bcp, -1);
3483 }
3484 
3485 //-----------------------------------------------------------------------------
3486 // Calls
3487 
3488 void TemplateTable::prepare_invoke(int byte_no,
3489                                    Register method,  // linked method (or i-klass)
3490                                    Register index,   // itable index, MethodType, etc.
3491                                    Register recv,    // If caller wants to see it.
3492                                    Register flags) { // If caller wants to test it.
3493   // Determine flags.
3494   const Bytecodes::Code code = bytecode();
3495   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3496   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3497   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3498   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3499   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3500   const bool load_receiver       = (recv != noreg);
3501   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3502 
3503   // Setup registers & access constant pool cache.
3504   if (recv  == noreg) { recv  = Z_ARG1; }
3505   if (flags == noreg) { flags = Z_ARG2; }
3506   assert_different_registers(method, Z_R14, index, recv, flags);
3507 
3508   BLOCK_COMMENT("prepare_invoke {");
3509 
3510   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3511 
3512   // Maybe push appendix to arguments.
3513   if (is_invokedynamic || is_invokehandle) {
3514     Label L_no_push;
3515     Register resolved_reference = Z_R1_scratch;
3516     __ testbit(flags, ConstantPoolCacheEntry::has_appendix_shift);
3517     __ z_bfalse(L_no_push);
3518     // Push the appendix as a trailing parameter.
3519     // This must be done before we get the receiver,
3520     // since the parameter_size includes it.
3521     __ load_resolved_reference_at_index(resolved_reference, index);
3522     __ verify_oop(resolved_reference);
3523     __ push_ptr(resolved_reference);  // Push appendix (MethodType, CallSite, etc.).
3524     __ bind(L_no_push);
3525   }
3526 
3527   // Load receiver if needed (after appendix is pushed so parameter size is correct).
3528   if (load_receiver) {
3529     assert(!is_invokedynamic, "");
3530     // recv := int2long(flags & ConstantPoolCacheEntry::parameter_size_mask) << 3
3531     // Flags is zero-extended int2long when loaded during load_invoke_cp_cache_entry().
3532     // Only the least significant byte (psize) of flags is used.
3533     {
3534       const unsigned int logSES = Interpreter::logStackElementSize;
3535       const int bit_shift = logSES;
3536       const int r_bitpos  = 63 - bit_shift;
3537       const int l_bitpos  = r_bitpos - ConstantPoolCacheEntry::parameter_size_bits + 1;
3538       const int n_rotate  = bit_shift;
3539       assert(ConstantPoolCacheEntry::parameter_size_mask == 255, "adapt bitpositions");
3540       __ rotate_then_insert(recv, flags, l_bitpos, r_bitpos, n_rotate, true);
3541     }
3542     // Recv now contains #arguments * StackElementSize.
3543 
3544     Address recv_addr(Z_esp, recv);
3545     __ z_lg(recv, recv_addr);
3546     __ verify_oop(recv);
3547   }
3548 
3549   // Compute return type.
3550   // ret_type is used by callers (invokespecial, invokestatic) at least.
3551   Register ret_type = Z_R1_scratch;
3552   assert_different_registers(ret_type, method);
3553 
3554   const address table_addr = (address)Interpreter::invoke_return_entry_table_for(code);
3555   __ load_absolute_address(Z_R14, table_addr);
3556 
3557   {
3558     const int bit_shift = LogBytesPerWord;           // Size of each table entry.
3559     const int r_bitpos  = 63 - bit_shift;
3560     const int l_bitpos  = r_bitpos - ConstantPoolCacheEntry::tos_state_bits + 1;
3561     const int n_rotate  = bit_shift-ConstantPoolCacheEntry::tos_state_shift;
3562     __ rotate_then_insert(ret_type, flags, l_bitpos, r_bitpos, n_rotate, true);
3563     // Make sure we don't need to mask flags for tos_state after the above shift.
3564     ConstantPoolCacheEntry::verify_tos_state_shift();
3565   }
3566 
3567     __ z_lg(Z_R14, Address(Z_R14, ret_type)); // Load return address.
3568   BLOCK_COMMENT("} prepare_invoke");
3569 }
3570 
3571 
3572 void TemplateTable::invokevirtual_helper(Register index,
3573                                          Register recv,
3574                                          Register flags) {
3575   // Uses temporary registers Z_tmp_2, Z_ARG4.
3576   assert_different_registers(index, recv, Z_tmp_2, Z_ARG4);
3577 
3578   // Test for an invoke of a final method.
3579   Label notFinal;
3580 
3581   BLOCK_COMMENT("invokevirtual_helper {");
3582 
3583   __ testbit(flags, ConstantPoolCacheEntry::is_vfinal_shift);
3584   __ z_brz(notFinal);
3585 
3586   const Register method = index;  // Method must be Z_ARG3.
3587   assert(method == Z_ARG3, "method must be second argument for interpreter calling convention");
3588 
3589   // Do the call - the index is actually the method to call.
3590   // That is, f2 is a vtable index if !is_vfinal, else f2 is a method.
3591 
3592   // It's final, need a null check here!
3593   __ null_check(recv);
3594 
3595   // Profile this call.
3596   __ profile_final_call(Z_tmp_2);
3597   __ profile_arguments_type(Z_tmp_2, method, Z_ARG5, true); // Argument type profiling.
3598   __ jump_from_interpreted(method, Z_tmp_2);
3599 
3600   __ bind(notFinal);
3601 
3602   // Get receiver klass.
3603   __ null_check(recv, Z_R0_scratch, oopDesc::klass_offset_in_bytes());
3604   __ load_klass(Z_tmp_2, recv);
3605 
3606   // Profile this call.
3607   __ profile_virtual_call(Z_tmp_2, Z_ARG4, Z_ARG5);
3608 
3609   // Get target method & entry point.
3610   __ z_sllg(index, index, exact_log2(vtableEntry::size_in_bytes()));
3611   __ mem2reg_opt(method,
3612                  Address(Z_tmp_2, index,
3613                          Klass::vtable_start_offset() + in_ByteSize(vtableEntry::method_offset_in_bytes())));
3614   __ profile_arguments_type(Z_ARG4, method, Z_ARG5, true);
3615   __ jump_from_interpreted(method, Z_ARG4);
3616   BLOCK_COMMENT("} invokevirtual_helper");
3617 }
3618 
3619 void TemplateTable::invokevirtual(int byte_no) {
3620   transition(vtos, vtos);
3621 
3622   assert(byte_no == f2_byte, "use this argument");
3623   prepare_invoke(byte_no,
3624                  Z_ARG3,  // method or vtable index
3625                  noreg,   // unused itable index
3626                  Z_ARG1,  // recv
3627                  Z_ARG2); // flags
3628 
3629   // Z_ARG3 : index
3630   // Z_ARG1 : receiver
3631   // Z_ARG2 : flags
3632   invokevirtual_helper(Z_ARG3, Z_ARG1, Z_ARG2);
3633 }
3634 
3635 void TemplateTable::invokespecial(int byte_no) {
3636   transition(vtos, vtos);
3637 
3638   assert(byte_no == f1_byte, "use this argument");
3639   Register Rmethod = Z_tmp_2;
3640   prepare_invoke(byte_no, Rmethod, noreg, // Get f1 method.
3641                  Z_ARG3);   // Get receiver also for null check.
3642   __ verify_oop(Z_ARG3);
3643   __ null_check(Z_ARG3);
3644   // Do the call.
3645   __ profile_call(Z_ARG2);
3646   __ profile_arguments_type(Z_ARG2, Rmethod, Z_ARG5, false);
3647   __ jump_from_interpreted(Rmethod, Z_R1_scratch);
3648 }
3649 
3650 void TemplateTable::invokestatic(int byte_no) {
3651   transition(vtos, vtos);
3652 
3653   assert(byte_no == f1_byte, "use this argument");
3654   Register Rmethod = Z_tmp_2;
3655   prepare_invoke(byte_no, Rmethod);   // Get f1 method.
3656   // Do the call.
3657   __ profile_call(Z_ARG2);
3658   __ profile_arguments_type(Z_ARG2, Rmethod, Z_ARG5, false);
3659   __ jump_from_interpreted(Rmethod, Z_R1_scratch);
3660 }
3661 
3662 // Outdated feature, and we don't support it.
3663 void TemplateTable::fast_invokevfinal(int byte_no) {
3664   transition(vtos, vtos);
3665   assert(byte_no == f2_byte, "use this argument");
3666   __ stop("fast_invokevfinal not used on linuxs390x");
3667 }
3668 
3669 void TemplateTable::invokeinterface(int byte_no) {
3670   transition(vtos, vtos);
3671 
3672   assert(byte_no == f1_byte, "use this argument");
3673   Register klass     = Z_ARG2,
3674            method    = Z_ARG3,
3675            interface = Z_ARG4,
3676            flags     = Z_ARG5,
3677            receiver  = Z_tmp_1;
3678 
3679   BLOCK_COMMENT("invokeinterface {");
3680 
3681   prepare_invoke(byte_no, interface, method,  // Get f1 klassOop, f2 itable index.
3682                  receiver, flags);
3683 
3684   // Z_R14 (== Z_bytecode) : return entry
3685 
3686   // Special case of invokeinterface called for virtual method of
3687   // java.lang.Object. See cpCacheOop.cpp for details.
3688   // This code isn't produced by javac, but could be produced by
3689   // another compliant java compiler.
3690   NearLabel notMethod, no_such_interface, no_such_method;
3691   __ testbit(flags, ConstantPoolCacheEntry::is_forced_virtual_shift);
3692   __ z_brz(notMethod);
3693   invokevirtual_helper(method, receiver, flags);
3694   __ bind(notMethod);
3695 
3696   // Get receiver klass into klass - also a null check.
3697   __ restore_locals();
3698   __ load_klass(klass, receiver);
3699 
3700   __ lookup_interface_method(klass, interface, noreg, noreg, /*temp*/Z_ARG1,
3701                              no_such_interface, /*return_method=*/false);
3702 
3703   // Profile this call.
3704   __ profile_virtual_call(klass, Z_ARG1/*mdp*/, flags/*scratch*/);
3705 
3706   // Find entry point to call.
3707 
3708   // Get declaring interface class from method
3709   __ z_lg(interface, Address(method, Method::const_offset()));
3710   __ z_lg(interface, Address(interface, ConstMethod::constants_offset()));
3711   __ z_lg(interface, Address(interface, ConstantPool::pool_holder_offset_in_bytes()));
3712 
3713   // Get itable index from method
3714   Register index   = receiver,
3715            method2 = flags;
3716   __ z_lgf(index, Address(method, Method::itable_index_offset()));
3717   __ z_aghi(index, -Method::itable_index_max);
3718   __ z_lcgr(index, index);
3719 
3720   __ lookup_interface_method(klass, interface, index, method2, Z_tmp_2,
3721                              no_such_interface);
3722 
3723   // Check for abstract method error.
3724   // Note: This should be done more efficiently via a throw_abstract_method_error
3725   // interpreter entry point and a conditional jump to it in case of a null
3726   // method.
3727   __ compareU64_and_branch(method2, (intptr_t) 0,
3728                             Assembler::bcondZero, no_such_method);
3729 
3730   __ profile_arguments_type(Z_tmp_1, method2, Z_tmp_2, true);
3731 
3732   // Do the call.
3733   __ jump_from_interpreted(method2, Z_tmp_2);
3734   __ should_not_reach_here();
3735 
3736   // exception handling code follows...
3737   // Note: Must restore interpreter registers to canonical
3738   // state for exception handling to work correctly!
3739 
3740   __ bind(no_such_method);
3741 
3742   // Throw exception.
3743   __ restore_bcp();      // Bcp must be correct for exception handler   (was destroyed).
3744   __ restore_locals();   // Make sure locals pointer is correct as well (was destroyed).
3745   __ call_VM(noreg,
3746              CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
3747   // The call_VM checks for exception, so we should never return here.
3748   __ should_not_reach_here();
3749 
3750   __ bind(no_such_interface);
3751 
3752   // Throw exception.
3753   __ restore_bcp();      // Bcp must be correct for exception handler   (was destroyed).
3754   __ restore_locals();   // Make sure locals pointer is correct as well (was destroyed).
3755   __ call_VM(noreg,
3756              CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_IncompatibleClassChangeError));
3757   // The call_VM checks for exception, so we should never return here.
3758   __ should_not_reach_here();
3759 
3760   BLOCK_COMMENT("} invokeinterface");
3761   return;
3762 }
3763 
3764 void TemplateTable::invokehandle(int byte_no) {
3765   transition(vtos, vtos);
3766 
3767   const Register method = Z_tmp_2;
3768   const Register recv   = Z_ARG5;
3769   const Register mtype  = Z_tmp_1;
3770   prepare_invoke(byte_no,
3771                  method, mtype,   // Get f2 method, f1 MethodType.
3772                  recv);
3773   __ verify_method_ptr(method);
3774   __ verify_oop(recv);
3775   __ null_check(recv);
3776 
3777   // Note: Mtype is already pushed (if necessary) by prepare_invoke.
3778 
3779   // FIXME: profile the LambdaForm also.
3780   __ profile_final_call(Z_ARG2);
3781   __ profile_arguments_type(Z_ARG3, method, Z_ARG5, true);
3782 
3783   __ jump_from_interpreted(method, Z_ARG3);
3784 }
3785 
3786 void TemplateTable::invokedynamic(int byte_no) {
3787   transition(vtos, vtos);
3788 
3789   const Register Rmethod   = Z_tmp_2;
3790   const Register Rcallsite = Z_tmp_1;
3791 
3792   prepare_invoke(byte_no, Rmethod, Rcallsite);
3793 
3794   // Rmethod: CallSite object (from f1)
3795   // Rcallsite: MH.linkToCallSite method (from f2)
3796 
3797   // Note: Callsite is already pushed by prepare_invoke.
3798 
3799   // TODO: should make a type profile for any invokedynamic that takes a ref argument.
3800   // Profile this call.
3801   __ profile_call(Z_ARG2);
3802   __ profile_arguments_type(Z_ARG2, Rmethod, Z_ARG5, false);
3803   __ jump_from_interpreted(Rmethod, Z_ARG2);
3804 }
3805 
3806 //-----------------------------------------------------------------------------
3807 // Allocation
3808 
3809 // Original comment on "allow_shared_alloc":
3810 // Always go the slow path.
3811 //  + Eliminated optimization within the template-based interpreter:
3812 //    If an allocation is done within the interpreter without using
3813 //    tlabs, the interpreter tries to do the allocation directly
3814 //    on the heap.
3815 //  + That means the profiling hooks are not considered and allocations
3816 //    get lost for the profiling framework.
3817 //  + However, we do not think that this optimization is really needed,
3818 //    so we always go now the slow path through the VM in this case --
3819 //    spec jbb2005 shows no measurable performance degradation.
3820 void TemplateTable::_new() {
3821   transition(vtos, atos);
3822   address prev_instr_address = NULL;
3823   Register tags  = Z_tmp_1;
3824   Register RallocatedObject   = Z_tos;
3825   Register cpool = Z_ARG2;
3826   Register tmp = Z_ARG3; // RobjectFields==tmp and Rsize==offset must be a register pair.
3827   Register offset = Z_ARG4;
3828   Label slow_case;
3829   Label done;
3830   Label initialize_header;
3831   Label allocate_shared;
3832 
3833   BLOCK_COMMENT("TemplateTable::_new {");
3834   __ get_2_byte_integer_at_bcp(offset/*dest*/, 1, InterpreterMacroAssembler::Unsigned);
3835   __ get_cpool_and_tags(cpool, tags);
3836   // Make sure the class we're about to instantiate has been resolved.
3837   // This is done before loading InstanceKlass to be consistent with the order
3838   // how Constant Pool is updated (see ConstantPool::klass_at_put).
3839   const int tags_offset = Array<u1>::base_offset_in_bytes();
3840   __ load_address(tmp, Address(tags, offset, tags_offset));
3841   __ z_cli(0, tmp, JVM_CONSTANT_Class);
3842   __ z_brne(slow_case);
3843 
3844   __ z_sllg(offset, offset, LogBytesPerWord); // Convert to to offset.
3845   // Get InstanceKlass.
3846   Register iklass = cpool;
3847   __ load_resolved_klass_at_offset(cpool, offset, iklass);
3848 
3849   // Make sure klass is initialized & doesn't have finalizer.
3850   // Make sure klass is fully initialized.
3851   const int state_offset = in_bytes(InstanceKlass::init_state_offset());
3852   if (Immediate::is_uimm12(state_offset)) {
3853     __ z_cli(state_offset, iklass, InstanceKlass::fully_initialized);
3854   } else {
3855     __ z_cliy(state_offset, iklass, InstanceKlass::fully_initialized);
3856   }
3857   __ z_brne(slow_case);
3858 
3859   // Get instance_size in InstanceKlass (scaled to a count of bytes).
3860   Register Rsize = offset;
3861   const int mask = 1 << Klass::_lh_instance_slow_path_bit;
3862   __ z_llgf(Rsize, Address(iklass, Klass::layout_helper_offset()));
3863   __ z_tmll(Rsize, mask);
3864   __ z_btrue(slow_case);
3865 
3866   // Allocate the instance
3867   // 1) Try to allocate in the TLAB.
3868   // 2) If the above fails (or is not applicable), go to a slow case
3869   // (creates a new TLAB, etc.).
3870   // Note: compared to other architectures, s390's implementation always goes
3871   // to the slow path if TLAB is used and fails.
3872   if (UseTLAB) {
3873     Register RoldTopValue = RallocatedObject;
3874     Register RnewTopValue = tmp;
3875     __ z_lg(RoldTopValue, Address(Z_thread, JavaThread::tlab_top_offset()));
3876     __ load_address(RnewTopValue, Address(RoldTopValue, Rsize));
3877     __ z_cg(RnewTopValue, Address(Z_thread, JavaThread::tlab_end_offset()));
3878     __ z_brh(slow_case);
3879     __ z_stg(RnewTopValue, Address(Z_thread, JavaThread::tlab_top_offset()));
3880 
3881     Register RobjectFields = tmp;
3882     Register Rzero = Z_R1_scratch;
3883     __ clear_reg(Rzero, true /*whole reg*/, false); // Load 0L into Rzero. Don't set CC.
3884 
3885     if (!ZeroTLAB) {
3886       // The object is initialized before the header. If the object size is
3887       // zero, go directly to the header initialization.
3888       __ z_aghi(Rsize, (int)-sizeof(oopDesc)); // Subtract header size, set CC.
3889       __ z_bre(initialize_header);             // Jump if size of fields is zero.
3890 
3891       // Initialize object fields.
3892       // See documentation for MVCLE instruction!!!
3893       assert(RobjectFields->encoding() % 2 == 0, "RobjectFields must be an even register");
3894       assert(Rsize->encoding() == (RobjectFields->encoding()+1),
3895              "RobjectFields and Rsize must be a register pair");
3896       assert(Rzero->encoding() % 2 == 1, "Rzero must be an odd register");
3897 
3898       // Set Rzero to 0 and use it as src length, then mvcle will copy nothing
3899       // and fill the object with the padding value 0.
3900       __ add2reg(RobjectFields, sizeof(oopDesc), RallocatedObject);
3901       __ move_long_ext(RobjectFields, as_Register(Rzero->encoding() - 1), 0);
3902     }
3903 
3904     // Initialize object header only.
3905     __ bind(initialize_header);
3906     if (UseBiasedLocking) {
3907       Register prototype = RobjectFields;
3908       __ z_lg(prototype, Address(iklass, Klass::prototype_header_offset()));
3909       __ z_stg(prototype, Address(RallocatedObject, oopDesc::mark_offset_in_bytes()));
3910     } else {
3911       __ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()),
3912                      (long)markOopDesc::prototype());
3913     }
3914 
3915     __ store_klass_gap(Rzero, RallocatedObject);  // Zero klass gap for compressed oops.
3916     __ store_klass(iklass, RallocatedObject);     // Store klass last.
3917 
3918     {
3919       SkipIfEqual skip(_masm, &DTraceAllocProbes, false, Z_ARG5 /*scratch*/);
3920       // Trigger dtrace event for fastpath.
3921       __ push(atos); // Save the return value.
3922       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), RallocatedObject);
3923       __ pop(atos); // Restore the return value.
3924     }
3925     __ z_bru(done);
3926   }
3927 
3928   // slow case
3929   __ bind(slow_case);
3930   __ get_constant_pool(Z_ARG2);
3931   __ get_2_byte_integer_at_bcp(Z_ARG3/*dest*/, 1, InterpreterMacroAssembler::Unsigned);
3932   call_VM(Z_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), Z_ARG2, Z_ARG3);
3933   __ verify_oop(Z_tos);
3934 
3935   // continue
3936   __ bind(done);
3937 
3938   BLOCK_COMMENT("} TemplateTable::_new");
3939 }
3940 
3941 void TemplateTable::newarray() {
3942   transition(itos, atos);
3943 
3944   // Call runtime.
3945   __ z_llgc(Z_ARG2, at_bcp(1));   // type
3946   __ z_lgfr(Z_ARG3, Z_tos);       // size
3947   call_VM(Z_RET,
3948           CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3949           Z_ARG2, Z_ARG3);
3950 }
3951 
3952 void TemplateTable::anewarray() {
3953   transition(itos, atos);
3954   __ get_2_byte_integer_at_bcp(Z_ARG3, 1, InterpreterMacroAssembler::Unsigned);
3955   __ get_constant_pool(Z_ARG2);
3956   __ z_lgfr(Z_ARG4, Z_tos);
3957   call_VM(Z_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3958           Z_ARG2, Z_ARG3, Z_ARG4);
3959 }
3960 
3961 void TemplateTable::arraylength() {
3962   transition(atos, itos);
3963 
3964   int offset = arrayOopDesc::length_offset_in_bytes();
3965 
3966   __ null_check(Z_tos, Z_R0_scratch, offset);
3967   __ mem2reg_opt(Z_tos, Address(Z_tos, offset), false);
3968 }
3969 
3970 void TemplateTable::checkcast() {
3971   transition(atos, atos);
3972 
3973   NearLabel done, is_null, ok_is_subtype, quicked, resolved;
3974 
3975   BLOCK_COMMENT("checkcast {");
3976   // If object is NULL, we are almost done.
3977   __ compareU64_and_branch(Z_tos, (intptr_t) 0, Assembler::bcondZero, is_null);
3978 
3979   // Get cpool & tags index.
3980   Register cpool = Z_tmp_1;
3981   Register tags = Z_tmp_2;
3982   Register index = Z_ARG5;
3983 
3984   __ get_cpool_and_tags(cpool, tags);
3985   __ get_2_byte_integer_at_bcp(index, 1, InterpreterMacroAssembler::Unsigned);
3986   // See if bytecode has already been quicked.
3987   // Note: For CLI, we would have to add the index to the tags pointer first,
3988   // thus load and compare in a "classic" manner.
3989   __ z_llgc(Z_R0_scratch,
3990             Address(tags, index, Array<u1>::base_offset_in_bytes()));
3991   __ compareU64_and_branch(Z_R0_scratch, JVM_CONSTANT_Class,
3992                            Assembler::bcondEqual, quicked);
3993 
3994   __ push(atos); // Save receiver for result, and for GC.
3995   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3996   __ get_vm_result_2(Z_tos);
3997 
3998   Register   receiver = Z_ARG4;
3999   Register   klass = Z_tos;
4000   Register   subklass = Z_ARG5;
4001 
4002   __ pop_ptr(receiver); // restore receiver
4003   __ z_bru(resolved);
4004 
4005   // Get superklass in klass and subklass in subklass.
4006   __ bind(quicked);
4007 
4008   __ z_lgr(Z_ARG4, Z_tos);  // Save receiver.
4009   __ z_sllg(index, index, LogBytesPerWord);  // index2bytes for addressing
4010   __ load_resolved_klass_at_offset(cpool, index, klass);
4011 
4012   __ bind(resolved);
4013 
4014   __ load_klass(subklass, receiver);
4015 
4016   // Generate subtype check. Object in receiver.
4017   // Superklass in klass. Subklass in subklass.
4018   __ gen_subtype_check(subklass, klass, Z_ARG3, Z_tmp_1, ok_is_subtype);
4019 
4020   // Come here on failure.
4021   __ push_ptr(receiver);
4022   // Object is at TOS, target klass oop expected in rax by convention.
4023   __ z_brul((address) Interpreter::_throw_ClassCastException_entry);
4024 
4025   // Come here on success.
4026   __ bind(ok_is_subtype);
4027 
4028   __ z_lgr(Z_tos, receiver); // Restore object.
4029 
4030   // Collect counts on whether this test sees NULLs a lot or not.
4031   if (ProfileInterpreter) {
4032     __ z_bru(done);
4033     __ bind(is_null);
4034     __ profile_null_seen(Z_tmp_1);
4035   } else {
4036     __ bind(is_null);   // Same as 'done'.
4037   }
4038 
4039   __ bind(done);
4040   BLOCK_COMMENT("} checkcast");
4041 }
4042 
4043 void TemplateTable::instanceof() {
4044   transition(atos, itos);
4045 
4046   NearLabel done, is_null, ok_is_subtype, quicked, resolved;
4047 
4048   BLOCK_COMMENT("instanceof {");
4049   // If object is NULL, we are almost done.
4050   __ compareU64_and_branch(Z_tos, (intptr_t) 0, Assembler::bcondZero, is_null);
4051 
4052   // Get cpool & tags index.
4053   Register cpool = Z_tmp_1;
4054   Register tags = Z_tmp_2;
4055   Register index = Z_ARG5;
4056 
4057   __ get_cpool_and_tags(cpool, tags);
4058   __ get_2_byte_integer_at_bcp(index, 1, InterpreterMacroAssembler::Unsigned);
4059   // See if bytecode has already been quicked.
4060   // Note: For CLI, we would have to add the index to the tags pointer first,
4061   // thus load and compare in a "classic" manner.
4062   __ z_llgc(Z_R0_scratch,
4063             Address(tags, index, Array<u1>::base_offset_in_bytes()));
4064   __ compareU64_and_branch(Z_R0_scratch, JVM_CONSTANT_Class, Assembler::bcondEqual, quicked);
4065 
4066   __ push(atos); // Save receiver for result, and for GC.
4067   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4068   __ get_vm_result_2(Z_tos);
4069 
4070   Register receiver = Z_tmp_2;
4071   Register klass = Z_tos;
4072   Register subklass = Z_tmp_2;
4073 
4074   __ pop_ptr(receiver); // Restore receiver.
4075   __ verify_oop(receiver);
4076   __ load_klass(subklass, subklass);
4077   __ z_bru(resolved);
4078 
4079   // Get superklass in klass and subklass in subklass.
4080   __ bind(quicked);
4081 
4082   __ load_klass(subklass, Z_tos);
4083   __ z_sllg(index, index, LogBytesPerWord);  // index2bytes for addressing
4084   __ load_resolved_klass_at_offset(cpool, index, klass);
4085 
4086   __ bind(resolved);
4087 
4088   // Generate subtype check.
4089   // Superklass in klass. Subklass in subklass.
4090   __ gen_subtype_check(subklass, klass, Z_ARG4, Z_ARG5, ok_is_subtype);
4091 
4092   // Come here on failure.
4093   __ clear_reg(Z_tos, true, false);
4094   __ z_bru(done);
4095 
4096   // Come here on success.
4097   __ bind(ok_is_subtype);
4098   __ load_const_optimized(Z_tos, 1);
4099 
4100   // Collect counts on whether this test sees NULLs a lot or not.
4101   if (ProfileInterpreter) {
4102     __ z_bru(done);
4103     __ bind(is_null);
4104     __ profile_null_seen(Z_tmp_1);
4105   } else {
4106     __ bind(is_null);   // same as 'done'
4107   }
4108 
4109   __ bind(done);
4110   // tos = 0: obj == NULL or  obj is not an instanceof the specified klass
4111   // tos = 1: obj != NULL and obj is     an instanceof the specified klass
4112   BLOCK_COMMENT("} instanceof");
4113 }
4114 
4115 //-----------------------------------------------------------------------------
4116 // Breakpoints
4117 void TemplateTable::_breakpoint() {
4118 
4119   // Note: We get here even if we are single stepping.
4120   // Jbug insists on setting breakpoints at every bytecode
4121   // even if we are in single step mode.
4122 
4123   transition(vtos, vtos);
4124 
4125   // Get the unpatched byte code.
4126   __ get_method(Z_ARG2);
4127   __ call_VM(noreg,
4128              CAST_FROM_FN_PTR(address, InterpreterRuntime::get_original_bytecode_at),
4129              Z_ARG2, Z_bcp);
4130   // Save the result to a register that is preserved over C-function calls.
4131   __ z_lgr(Z_tmp_1, Z_RET);
4132 
4133   // Post the breakpoint event.
4134   __ get_method(Z_ARG2);
4135   __ call_VM(noreg,
4136              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
4137              Z_ARG2, Z_bcp);
4138 
4139   // Must restore the bytecode, because call_VM destroys Z_bytecode.
4140   __ z_lgr(Z_bytecode, Z_tmp_1);
4141 
4142   // Complete the execution of original bytecode.
4143   __ dispatch_only_normal(vtos);
4144 }
4145 
4146 
4147 // Exceptions
4148 
4149 void TemplateTable::athrow() {
4150   transition(atos, vtos);
4151   __ null_check(Z_tos);
4152   __ load_absolute_address(Z_ARG2, Interpreter::throw_exception_entry());
4153   __ z_br(Z_ARG2);
4154 }
4155 
4156 // Synchronization
4157 //
4158 // Note: monitorenter & exit are symmetric routines; which is reflected
4159 //       in the assembly code structure as well
4160 //
4161 // Stack layout:
4162 //
4163 //               callers_sp        <- Z_SP (callers_sp == Z_fp (own fp))
4164 //               return_pc
4165 //               [rest of ABI_160]
4166 //              /slot o:   free
4167 //             / ...       free
4168 //       oper. | slot n+1: free    <- Z_esp points to first free slot
4169 //       stack | slot n:   val                      caches IJAVA_STATE.esp
4170 //             | ...
4171 //              \slot 0:   val
4172 //              /slot m            <- IJAVA_STATE.monitors = monitor block top
4173 //             | ...
4174 //     monitors| slot 2
4175 //             | slot 1
4176 //              \slot 0
4177 //              /slot l            <- monitor block bot
4178 // ijava_state | ...
4179 //             | slot 2
4180 //              \slot 0
4181 //                                 <- Z_fp
4182 void TemplateTable::monitorenter() {
4183   transition(atos, vtos);
4184 
4185   BLOCK_COMMENT("monitorenter {");
4186 
4187   // Check for NULL object.
4188   __ null_check(Z_tos);
4189   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4190   NearLabel allocated;
4191   // Initialize entry pointer.
4192   const Register Rfree_slot = Z_tmp_1;
4193   __ clear_reg(Rfree_slot, true, false); // Points to free slot or NULL. Don't set CC.
4194 
4195   // Find a free slot in the monitor block from top to bot (result in Rfree_slot).
4196   {
4197     const Register Rcurr_monitor = Z_ARG2;
4198     const Register Rbot = Z_ARG3; // Points to word under bottom of monitor block.
4199     const Register Rlocked_obj = Z_ARG4;
4200     NearLabel loop, exit, not_free;
4201     // Starting with top-most entry.
4202     __ get_monitors(Rcurr_monitor); // Rcur_monitor = IJAVA_STATE.monitors
4203     __ add2reg(Rbot, -frame::z_ijava_state_size, Z_fp);
4204 
4205 #ifdef ASSERT
4206     address reentry = NULL;
4207     { NearLabel ok;
4208       __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondNotHigh, ok);
4209       reentry = __ stop_chain_static(reentry, "IJAVA_STATE.monitors points below monitor block bottom");
4210       __ bind(ok);
4211     }
4212     { NearLabel ok;
4213       __ compareU64_and_branch(Rcurr_monitor, Z_esp, Assembler::bcondHigh, ok);
4214       reentry = __ stop_chain_static(reentry, "IJAVA_STATE.monitors above Z_esp");
4215       __ bind(ok);
4216     }
4217 #endif
4218 
4219     // Check if bottom reached, i.e. if there is at least one monitor.
4220     __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondEqual, exit);
4221 
4222     __ bind(loop);
4223     // Check if current entry is used.
4224     __ load_and_test_long(Rlocked_obj, Address(Rcurr_monitor, BasicObjectLock::obj_offset_in_bytes()));
4225     __ z_brne(not_free);
4226     // If not used then remember entry in Rfree_slot.
4227     __ z_lgr(Rfree_slot, Rcurr_monitor);
4228     __ bind(not_free);
4229     // Exit if current entry is for same object; this guarantees, that new monitor
4230     // used for recursive lock is above the older one.
4231     __ compareU64_and_branch(Rlocked_obj, Z_tos, Assembler::bcondEqual, exit);
4232     // otherwise advance to next entry
4233     __ add2reg(Rcurr_monitor, entry_size);
4234     // Check if bottom reached, if not at bottom then check this entry.
4235     __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondNotEqual, loop);
4236     __ bind(exit);
4237   }
4238 
4239   // Rfree_slot != NULL -> found one
4240   __ compareU64_and_branch(Rfree_slot, (intptr_t)0L, Assembler::bcondNotEqual, allocated);
4241 
4242   // Allocate one if there's no free slot.
4243   __ add_monitor_to_stack(false, Z_ARG3, Z_ARG4, Z_ARG5);
4244   __ get_monitors(Rfree_slot);
4245 
4246   // Rfree_slot: points to monitor entry.
4247   __ bind(allocated);
4248 
4249   // Increment bcp to point to the next bytecode, so exception
4250   // handling for async. exceptions work correctly.
4251   // The object has already been poped from the stack, so the
4252   // expression stack looks correct.
4253   __ add2reg(Z_bcp, 1, Z_bcp);
4254 
4255   // Store object.
4256   __ z_stg(Z_tos, BasicObjectLock::obj_offset_in_bytes(), Rfree_slot);
4257   __ lock_object(Rfree_slot, Z_tos);
4258 
4259   // Check to make sure this monitor doesn't cause stack overflow after locking.
4260   __ save_bcp();  // in case of exception
4261   __ generate_stack_overflow_check(0);
4262 
4263   // The bcp has already been incremented. Just need to dispatch to
4264   // next instruction.
4265   __ dispatch_next(vtos);
4266 
4267   BLOCK_COMMENT("} monitorenter");
4268 }
4269 
4270 
4271 void TemplateTable::monitorexit() {
4272   transition(atos, vtos);
4273 
4274   BLOCK_COMMENT("monitorexit {");
4275 
4276   // Check for NULL object.
4277   __ null_check(Z_tos);
4278 
4279   NearLabel found, not_found;
4280   const Register Rcurr_monitor = Z_ARG2;
4281 
4282   // Find matching slot.
4283   {
4284     const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4285     NearLabel entry, loop;
4286 
4287     const Register Rbot = Z_ARG3; // Points to word under bottom of monitor block.
4288     const Register Rlocked_obj = Z_ARG4;
4289     // Starting with top-most entry.
4290     __ get_monitors(Rcurr_monitor); // Rcur_monitor = IJAVA_STATE.monitors
4291     __ add2reg(Rbot, -frame::z_ijava_state_size, Z_fp);
4292 
4293 #ifdef ASSERT
4294     address reentry = NULL;
4295     { NearLabel ok;
4296       __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondNotHigh, ok);
4297       reentry = __ stop_chain_static(reentry, "IJAVA_STATE.monitors points below monitor block bottom");
4298       __ bind(ok);
4299     }
4300     { NearLabel ok;
4301       __ compareU64_and_branch(Rcurr_monitor, Z_esp, Assembler::bcondHigh, ok);
4302       reentry = __ stop_chain_static(reentry, "IJAVA_STATE.monitors above Z_esp");
4303       __ bind(ok);
4304     }
4305 #endif
4306 
4307     // Check if bottom reached, i.e. if there is at least one monitor.
4308     __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondEqual, not_found);
4309 
4310     __ bind(loop);
4311     // Check if current entry is for same object.
4312     __ z_lg(Rlocked_obj, Address(Rcurr_monitor, BasicObjectLock::obj_offset_in_bytes()));
4313     // If same object then stop searching.
4314     __ compareU64_and_branch(Rlocked_obj, Z_tos, Assembler::bcondEqual, found);
4315     // Otherwise advance to next entry.
4316     __ add2reg(Rcurr_monitor, entry_size);
4317     // Check if bottom reached, if not at bottom then check this entry.
4318     __ compareU64_and_branch(Rcurr_monitor, Rbot, Assembler::bcondNotEqual, loop);
4319   }
4320 
4321   __ bind(not_found);
4322   // Error handling. Unlocking was not block-structured.
4323   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4324                    InterpreterRuntime::throw_illegal_monitor_state_exception));
4325   __ should_not_reach_here();
4326 
4327   __ bind(found);
4328   __ push_ptr(Z_tos); // Make sure object is on stack (contract with oopMaps).
4329   __ unlock_object(Rcurr_monitor, Z_tos);
4330   __ pop_ptr(Z_tos); // Discard object.
4331   BLOCK_COMMENT("} monitorexit");
4332 }
4333 
4334 // Wide instructions
4335 void TemplateTable::wide() {
4336   transition(vtos, vtos);
4337 
4338   __ z_llgc(Z_R1_scratch, at_bcp(1));
4339   __ z_sllg(Z_R1_scratch, Z_R1_scratch, LogBytesPerWord);
4340   __ load_absolute_address(Z_tmp_1, (address) Interpreter::_wentry_point);
4341   __ mem2reg_opt(Z_tmp_1, Address(Z_tmp_1, Z_R1_scratch));
4342   __ z_br(Z_tmp_1);
4343   // Note: the bcp increment step is part of the individual wide
4344   // bytecode implementations.
4345 }
4346 
4347 // Multi arrays
4348 void TemplateTable::multianewarray() {
4349   transition(vtos, atos);
4350 
4351   __ z_llgc(Z_tmp_1, at_bcp(3)); // Get number of dimensions.
4352   // Slot count to byte offset.
4353   __ z_sllg(Z_tmp_1, Z_tmp_1, Interpreter::logStackElementSize);
4354   // Z_esp points past last_dim, so set to Z_ARG2 to first_dim address.
4355   __ load_address(Z_ARG2, Address(Z_esp, Z_tmp_1));
4356   call_VM(Z_RET,
4357           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
4358           Z_ARG2);
4359   // Pop dimensions from expression stack.
4360   __ z_agr(Z_esp, Z_tmp_1);
4361 }