1 /*
   2  * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "interpreter/interp_masm.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "interpreter/interpreterRuntime.hpp"
  30 #include "interpreter/templateTable.hpp"
  31 #include "memory/universe.inline.hpp"
  32 #include "oops/cpCache.hpp"
  33 #include "oops/methodData.hpp"
  34 #include "oops/objArrayKlass.hpp"
  35 #include "oops/oop.inline.hpp"
  36 #include "prims/methodHandles.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "runtime/synchronizer.hpp"
  40 
  41 #define __ _masm->
  42 
  43 //----------------------------------------------------------------------------------------------------
  44 // Platform-dependent initialization
  45 
  46 void TemplateTable::pd_initialize() {
  47   // No arm specific initialization
  48 }
  49 
  50 //----------------------------------------------------------------------------------------------------
  51 // Address computation
  52 
  53 // local variables
  54 static inline Address iaddress(int n)            {
  55   return Address(Rlocals, Interpreter::local_offset_in_bytes(n));
  56 }
  57 
  58 static inline Address laddress(int n)            { return iaddress(n + 1); }
  59 #ifndef AARCH64
  60 static inline Address haddress(int n)            { return iaddress(n + 0); }
  61 #endif // !AARCH64
  62 
  63 static inline Address faddress(int n)            { return iaddress(n); }
  64 static inline Address daddress(int n)            { return laddress(n); }
  65 static inline Address aaddress(int n)            { return iaddress(n); }
  66 
  67 
  68 void TemplateTable::get_local_base_addr(Register r, Register index) {
  69   __ sub(r, Rlocals, AsmOperand(index, lsl, Interpreter::logStackElementSize));
  70 }
  71 
  72 Address TemplateTable::load_iaddress(Register index, Register scratch) {
  73 #ifdef AARCH64
  74   get_local_base_addr(scratch, index);
  75   return Address(scratch);
  76 #else
  77   return Address(Rlocals, index, lsl, Interpreter::logStackElementSize, basic_offset, sub_offset);
  78 #endif // AARCH64
  79 }
  80 
  81 Address TemplateTable::load_aaddress(Register index, Register scratch) {
  82   return load_iaddress(index, scratch);
  83 }
  84 
  85 Address TemplateTable::load_faddress(Register index, Register scratch) {
  86 #ifdef __SOFTFP__
  87   return load_iaddress(index, scratch);
  88 #else
  89   get_local_base_addr(scratch, index);
  90   return Address(scratch);
  91 #endif // __SOFTFP__
  92 }
  93 
  94 Address TemplateTable::load_daddress(Register index, Register scratch) {
  95   get_local_base_addr(scratch, index);
  96   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  97 }
  98 
  99 // At top of Java expression stack which may be different than SP.
 100 // It isn't for category 1 objects.
 101 static inline Address at_tos() {
 102   return Address(Rstack_top, Interpreter::expr_offset_in_bytes(0));
 103 }
 104 
 105 static inline Address at_tos_p1() {
 106   return Address(Rstack_top, Interpreter::expr_offset_in_bytes(1));
 107 }
 108 
 109 static inline Address at_tos_p2() {
 110   return Address(Rstack_top, Interpreter::expr_offset_in_bytes(2));
 111 }
 112 
 113 
 114 // 32-bit ARM:
 115 // Loads double/long local into R0_tos_lo/R1_tos_hi with two
 116 // separate ldr instructions (supports nonadjacent values).
 117 // Used for longs in all modes, and for doubles in SOFTFP mode.
 118 //
 119 // AArch64: loads long local into R0_tos.
 120 //
 121 void TemplateTable::load_category2_local(Register Rlocal_index, Register tmp) {
 122   const Register Rlocal_base = tmp;
 123   assert_different_registers(Rlocal_index, tmp);
 124 
 125   get_local_base_addr(Rlocal_base, Rlocal_index);
 126 #ifdef AARCH64
 127   __ ldr(R0_tos, Address(Rlocal_base, Interpreter::local_offset_in_bytes(1)));
 128 #else
 129   __ ldr(R0_tos_lo, Address(Rlocal_base, Interpreter::local_offset_in_bytes(1)));
 130   __ ldr(R1_tos_hi, Address(Rlocal_base, Interpreter::local_offset_in_bytes(0)));
 131 #endif // AARCH64
 132 }
 133 
 134 
 135 // 32-bit ARM:
 136 // Stores R0_tos_lo/R1_tos_hi to double/long local with two
 137 // separate str instructions (supports nonadjacent values).
 138 // Used for longs in all modes, and for doubles in SOFTFP mode
 139 //
 140 // AArch64: stores R0_tos to long local.
 141 //
 142 void TemplateTable::store_category2_local(Register Rlocal_index, Register tmp) {
 143   const Register Rlocal_base = tmp;
 144   assert_different_registers(Rlocal_index, tmp);
 145 
 146   get_local_base_addr(Rlocal_base, Rlocal_index);
 147 #ifdef AARCH64
 148   __ str(R0_tos, Address(Rlocal_base, Interpreter::local_offset_in_bytes(1)));
 149 #else
 150   __ str(R0_tos_lo, Address(Rlocal_base, Interpreter::local_offset_in_bytes(1)));
 151   __ str(R1_tos_hi, Address(Rlocal_base, Interpreter::local_offset_in_bytes(0)));
 152 #endif // AARCH64
 153 }
 154 
 155 // Returns address of Java array element using temp register as address base.
 156 Address TemplateTable::get_array_elem_addr(BasicType elemType, Register array, Register index, Register temp) {
 157   int logElemSize = exact_log2(type2aelembytes(elemType));
 158   __ add_ptr_scaled_int32(temp, array, index, logElemSize);
 159   return Address(temp, arrayOopDesc::base_offset_in_bytes(elemType));
 160 }
 161 
 162 //----------------------------------------------------------------------------------------------------
 163 // Condition conversion
 164 AsmCondition convNegCond(TemplateTable::Condition cc) {
 165   switch (cc) {
 166     case TemplateTable::equal        : return ne;
 167     case TemplateTable::not_equal    : return eq;
 168     case TemplateTable::less         : return ge;
 169     case TemplateTable::less_equal   : return gt;
 170     case TemplateTable::greater      : return le;
 171     case TemplateTable::greater_equal: return lt;
 172   }
 173   ShouldNotReachHere();
 174   return nv;
 175 }
 176 
 177 //----------------------------------------------------------------------------------------------------
 178 // Miscelaneous helper routines
 179 
 180 // Store an oop (or NULL) at the address described by obj.
 181 // Blows all volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
 182 // Also destroys new_val and obj.base().
 183 static void do_oop_store(InterpreterMacroAssembler* _masm,
 184                          Address obj,
 185                          Register new_val,
 186                          Register tmp1,
 187                          Register tmp2,
 188                          Register tmp3,
 189                          BarrierSet::Name barrier,
 190                          bool precise,
 191                          bool is_null) {
 192 
 193   assert_different_registers(obj.base(), new_val, tmp1, tmp2, tmp3, noreg);
 194   switch (barrier) {
 195 #if INCLUDE_ALL_GCS
 196     case BarrierSet::G1SATBCTLogging:
 197       {
 198         // flatten object address if needed
 199         assert (obj.mode() == basic_offset, "pre- or post-indexing is not supported here");
 200 
 201         const Register store_addr = obj.base();
 202         if (obj.index() != noreg) {
 203           assert (obj.disp() == 0, "index or displacement, not both");
 204 #ifdef AARCH64
 205           __ add(store_addr, obj.base(), obj.index(), obj.extend(), obj.shift_imm());
 206 #else
 207           assert(obj.offset_op() == add_offset, "addition is expected");
 208           __ add(store_addr, obj.base(), AsmOperand(obj.index(), obj.shift(), obj.shift_imm()));
 209 #endif // AARCH64
 210         } else if (obj.disp() != 0) {
 211           __ add(store_addr, obj.base(), obj.disp());
 212         }
 213 
 214         __ g1_write_barrier_pre(store_addr, new_val, tmp1, tmp2, tmp3);
 215         if (is_null) {
 216           __ store_heap_oop_null(new_val, Address(store_addr));
 217         } else {
 218           // G1 barrier needs uncompressed oop for region cross check.
 219           Register val_to_store = new_val;
 220           if (UseCompressedOops) {
 221             val_to_store = tmp1;
 222             __ mov(val_to_store, new_val);
 223           }
 224           __ store_heap_oop(val_to_store, Address(store_addr)); // blows val_to_store:
 225           val_to_store = noreg;
 226           __ g1_write_barrier_post(store_addr, new_val, tmp1, tmp2, tmp3);
 227         }
 228       }
 229       break;
 230 #endif // INCLUDE_ALL_GCS
 231     case BarrierSet::CardTableForRS:
 232     case BarrierSet::CardTableExtension:
 233       {
 234         if (is_null) {
 235           __ store_heap_oop_null(new_val, obj);
 236         } else {
 237           assert (!precise || (obj.index() == noreg && obj.disp() == 0),
 238                   "store check address should be calculated beforehand");
 239 
 240           __ store_check_part1(tmp1);
 241           __ store_heap_oop(new_val, obj); // blows new_val:
 242           new_val = noreg;
 243           __ store_check_part2(obj.base(), tmp1, tmp2);
 244         }
 245       }
 246       break;
 247     case BarrierSet::ModRef:
 248       ShouldNotReachHere();
 249       break;
 250     default:
 251       ShouldNotReachHere();
 252       break;
 253   }
 254 }
 255 
 256 Address TemplateTable::at_bcp(int offset) {
 257   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 258   return Address(Rbcp, offset);
 259 }
 260 
 261 
 262 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR.
 263 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 264                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 265                                    int byte_no) {
 266   assert_different_registers(bc_reg, temp_reg);
 267   if (!RewriteBytecodes)  return;
 268   Label L_patch_done;
 269 
 270   switch (bc) {
 271   case Bytecodes::_fast_aputfield:
 272   case Bytecodes::_fast_bputfield:
 273   case Bytecodes::_fast_zputfield:
 274   case Bytecodes::_fast_cputfield:
 275   case Bytecodes::_fast_dputfield:
 276   case Bytecodes::_fast_fputfield:
 277   case Bytecodes::_fast_iputfield:
 278   case Bytecodes::_fast_lputfield:
 279   case Bytecodes::_fast_sputfield:
 280     {
 281       // We skip bytecode quickening for putfield instructions when
 282       // the put_code written to the constant pool cache is zero.
 283       // This is required so that every execution of this instruction
 284       // calls out to InterpreterRuntime::resolve_get_put to do
 285       // additional, required work.
 286       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 287       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 288       __ get_cache_and_index_and_bytecode_at_bcp(bc_reg, temp_reg, temp_reg, byte_no, 1, sizeof(u2));
 289       __ mov(bc_reg, bc);
 290       __ cbz(temp_reg, L_patch_done);  // test if bytecode is zero
 291     }
 292     break;
 293   default:
 294     assert(byte_no == -1, "sanity");
 295     // the pair bytecodes have already done the load.
 296     if (load_bc_into_bc_reg) {
 297       __ mov(bc_reg, bc);
 298     }
 299   }
 300 
 301   if (__ can_post_breakpoint()) {
 302     Label L_fast_patch;
 303     // if a breakpoint is present we can't rewrite the stream directly
 304     __ ldrb(temp_reg, at_bcp(0));
 305     __ cmp(temp_reg, Bytecodes::_breakpoint);
 306     __ b(L_fast_patch, ne);
 307     if (bc_reg != R3) {
 308       __ mov(R3, bc_reg);
 309     }
 310     __ mov(R1, Rmethod);
 311     __ mov(R2, Rbcp);
 312     // Let breakpoint table handling rewrite to quicker bytecode
 313     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), R1, R2, R3);
 314     __ b(L_patch_done);
 315     __ bind(L_fast_patch);
 316   }
 317 
 318 #ifdef ASSERT
 319   Label L_okay;
 320   __ ldrb(temp_reg, at_bcp(0));
 321   __ cmp(temp_reg, (int)Bytecodes::java_code(bc));
 322   __ b(L_okay, eq);
 323   __ cmp(temp_reg, bc_reg);
 324   __ b(L_okay, eq);
 325   __ stop("patching the wrong bytecode");
 326   __ bind(L_okay);
 327 #endif
 328 
 329   // patch bytecode
 330   __ strb(bc_reg, at_bcp(0));
 331   __ bind(L_patch_done);
 332 }
 333 
 334 //----------------------------------------------------------------------------------------------------
 335 // Individual instructions
 336 
 337 void TemplateTable::nop() {
 338   transition(vtos, vtos);
 339   // nothing to do
 340 }
 341 
 342 void TemplateTable::shouldnotreachhere() {
 343   transition(vtos, vtos);
 344   __ stop("shouldnotreachhere bytecode");
 345 }
 346 
 347 
 348 
 349 void TemplateTable::aconst_null() {
 350   transition(vtos, atos);
 351   __ mov(R0_tos, 0);
 352 }
 353 
 354 
 355 void TemplateTable::iconst(int value) {
 356   transition(vtos, itos);
 357   __ mov_slow(R0_tos, value);
 358 }
 359 
 360 
 361 void TemplateTable::lconst(int value) {
 362   transition(vtos, ltos);
 363   assert((value == 0) || (value == 1), "unexpected long constant");
 364   __ mov(R0_tos, value);
 365 #ifndef AARCH64
 366   __ mov(R1_tos_hi, 0);
 367 #endif // !AARCH64
 368 }
 369 
 370 
 371 void TemplateTable::fconst(int value) {
 372   transition(vtos, ftos);
 373 #ifdef AARCH64
 374   switch(value) {
 375   case 0:   __ fmov_sw(S0_tos, ZR);    break;
 376   case 1:   __ fmov_s (S0_tos, 0x70);  break;
 377   case 2:   __ fmov_s (S0_tos, 0x00);  break;
 378   default:  ShouldNotReachHere();      break;
 379   }
 380 #else
 381   const int zero = 0;         // 0.0f
 382   const int one = 0x3f800000; // 1.0f
 383   const int two = 0x40000000; // 2.0f
 384 
 385   switch(value) {
 386   case 0:   __ mov(R0_tos, zero);   break;
 387   case 1:   __ mov(R0_tos, one);    break;
 388   case 2:   __ mov(R0_tos, two);    break;
 389   default:  ShouldNotReachHere();   break;
 390   }
 391 
 392 #ifndef __SOFTFP__
 393   __ fmsr(S0_tos, R0_tos);
 394 #endif // !__SOFTFP__
 395 #endif // AARCH64
 396 }
 397 
 398 
 399 void TemplateTable::dconst(int value) {
 400   transition(vtos, dtos);
 401 #ifdef AARCH64
 402   switch(value) {
 403   case 0:   __ fmov_dx(D0_tos, ZR);    break;
 404   case 1:   __ fmov_d (D0_tos, 0x70);  break;
 405   default:  ShouldNotReachHere();      break;
 406   }
 407 #else
 408   const int one_lo = 0;            // low part of 1.0
 409   const int one_hi = 0x3ff00000;   // high part of 1.0
 410 
 411   if (value == 0) {
 412 #ifdef __SOFTFP__
 413     __ mov(R0_tos_lo, 0);
 414     __ mov(R1_tos_hi, 0);
 415 #else
 416     __ mov(R0_tmp, 0);
 417     __ fmdrr(D0_tos, R0_tmp, R0_tmp);
 418 #endif // __SOFTFP__
 419   } else if (value == 1) {
 420     __ mov(R0_tos_lo, one_lo);
 421     __ mov_slow(R1_tos_hi, one_hi);
 422 #ifndef __SOFTFP__
 423     __ fmdrr(D0_tos, R0_tos_lo, R1_tos_hi);
 424 #endif // !__SOFTFP__
 425   } else {
 426     ShouldNotReachHere();
 427   }
 428 #endif // AARCH64
 429 }
 430 
 431 
 432 void TemplateTable::bipush() {
 433   transition(vtos, itos);
 434   __ ldrsb(R0_tos, at_bcp(1));
 435 }
 436 
 437 
 438 void TemplateTable::sipush() {
 439   transition(vtos, itos);
 440   __ ldrsb(R0_tmp, at_bcp(1));
 441   __ ldrb(R1_tmp, at_bcp(2));
 442   __ orr(R0_tos, R1_tmp, AsmOperand(R0_tmp, lsl, BitsPerByte));
 443 }
 444 
 445 
 446 void TemplateTable::ldc(bool wide) {
 447   transition(vtos, vtos);
 448   Label fastCase, Done;
 449 
 450   const Register Rindex = R1_tmp;
 451   const Register Rcpool = R2_tmp;
 452   const Register Rtags  = R3_tmp;
 453   const Register RtagType = R3_tmp;
 454 
 455   if (wide) {
 456     __ get_unsigned_2_byte_index_at_bcp(Rindex, 1);
 457   } else {
 458     __ ldrb(Rindex, at_bcp(1));
 459   }
 460   __ get_cpool_and_tags(Rcpool, Rtags);
 461 
 462   const int base_offset = ConstantPool::header_size() * wordSize;
 463   const int tags_offset = Array<u1>::base_offset_in_bytes();
 464 
 465   // get const type
 466   __ add(Rtemp, Rtags, tags_offset);
 467 #ifdef AARCH64
 468   __ add(Rtemp, Rtemp, Rindex);
 469   __ ldarb(RtagType, Rtemp);  // TODO-AARCH64 figure out if barrier is needed here, or control dependency is enough
 470 #else
 471   __ ldrb(RtagType, Address(Rtemp, Rindex));
 472   volatile_barrier(MacroAssembler::LoadLoad, Rtemp);
 473 #endif // AARCH64
 474 
 475   // unresolved class - get the resolved class
 476   __ cmp(RtagType, JVM_CONSTANT_UnresolvedClass);
 477 
 478   // unresolved class in error (resolution failed) - call into runtime
 479   // so that the same error from first resolution attempt is thrown.
 480 #ifdef AARCH64
 481   __ mov(Rtemp, JVM_CONSTANT_UnresolvedClassInError); // this constant does not fit into 5-bit immediate constraint
 482   __ cond_cmp(RtagType, Rtemp, ne);
 483 #else
 484   __ cond_cmp(RtagType, JVM_CONSTANT_UnresolvedClassInError, ne);
 485 #endif // AARCH64
 486 
 487   // resolved class - need to call vm to get java mirror of the class
 488   __ cond_cmp(RtagType, JVM_CONSTANT_Class, ne);
 489 
 490   __ b(fastCase, ne);
 491 
 492   // slow case - call runtime
 493   __ mov(R1, wide);
 494   call_VM(R0_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), R1);
 495   __ push(atos);
 496   __ b(Done);
 497 
 498   // int, float, String
 499   __ bind(fastCase);
 500 #ifdef ASSERT
 501   { Label L;
 502     __ cmp(RtagType, JVM_CONSTANT_Integer);
 503     __ cond_cmp(RtagType, JVM_CONSTANT_Float, ne);
 504     __ b(L, eq);
 505     __ stop("unexpected tag type in ldc");
 506     __ bind(L);
 507   }
 508 #endif // ASSERT
 509   // itos, ftos
 510   __ add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord));
 511   __ ldr_u32(R0_tos, Address(Rtemp, base_offset));
 512 
 513   // floats and ints are placed on stack in the same way, so
 514   // we can use push(itos) to transfer float value without VFP
 515   __ push(itos);
 516   __ bind(Done);
 517 }
 518 
 519 // Fast path for caching oop constants.
 520 void TemplateTable::fast_aldc(bool wide) {
 521   transition(vtos, atos);
 522   int index_size = wide ? sizeof(u2) : sizeof(u1);
 523   Label resolved;
 524 
 525   // We are resolved if the resolved reference cache entry contains a
 526   // non-null object (CallSite, etc.)
 527   assert_different_registers(R0_tos, R2_tmp);
 528   __ get_index_at_bcp(R2_tmp, 1, R0_tos, index_size);
 529   __ load_resolved_reference_at_index(R0_tos, R2_tmp);
 530   __ cbnz(R0_tos, resolved);
 531 
 532   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 533 
 534   // first time invocation - must resolve first
 535   __ mov(R1, (int)bytecode());
 536   __ call_VM(R0_tos, entry, R1);
 537   __ bind(resolved);
 538 
 539   if (VerifyOops) {
 540     __ verify_oop(R0_tos);
 541   }
 542 }
 543 
 544 void TemplateTable::ldc2_w() {
 545   transition(vtos, vtos);
 546   const Register Rtags  = R2_tmp;
 547   const Register Rindex = R3_tmp;
 548   const Register Rcpool = R4_tmp;
 549   const Register Rbase  = R5_tmp;
 550 
 551   __ get_unsigned_2_byte_index_at_bcp(Rindex, 1);
 552 
 553   __ get_cpool_and_tags(Rcpool, Rtags);
 554   const int base_offset = ConstantPool::header_size() * wordSize;
 555   const int tags_offset = Array<u1>::base_offset_in_bytes();
 556 
 557   __ add(Rbase, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord));
 558 
 559 #ifdef __ABI_HARD__
 560   Label Long, exit;
 561   // get type from tags
 562   __ add(Rtemp, Rtags, tags_offset);
 563   __ ldrb(Rtemp, Address(Rtemp, Rindex));
 564   __ cmp(Rtemp, JVM_CONSTANT_Double);
 565   __ b(Long, ne);
 566   __ ldr_double(D0_tos, Address(Rbase, base_offset));
 567 
 568   __ push(dtos);
 569   __ b(exit);
 570   __ bind(Long);
 571 #endif
 572 
 573 #ifdef AARCH64
 574   __ ldr(R0_tos, Address(Rbase, base_offset));
 575 #else
 576   __ ldr(R0_tos_lo, Address(Rbase, base_offset + 0 * wordSize));
 577   __ ldr(R1_tos_hi, Address(Rbase, base_offset + 1 * wordSize));
 578 #endif // AARCH64
 579   __ push(ltos);
 580 
 581 #ifdef __ABI_HARD__
 582   __ bind(exit);
 583 #endif
 584 }
 585 
 586 
 587 void TemplateTable::locals_index(Register reg, int offset) {
 588   __ ldrb(reg, at_bcp(offset));
 589 }
 590 
 591 void TemplateTable::iload() {
 592   iload_internal();
 593 }
 594 
 595 void TemplateTable::nofast_iload() {
 596   iload_internal(may_not_rewrite);
 597 }
 598 
 599 void TemplateTable::iload_internal(RewriteControl rc) {
 600   transition(vtos, itos);
 601 
 602   if ((rc == may_rewrite) && __ rewrite_frequent_pairs()) {
 603     Label rewrite, done;
 604     const Register next_bytecode = R1_tmp;
 605     const Register target_bytecode = R2_tmp;
 606 
 607     // get next byte
 608     __ ldrb(next_bytecode, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 609     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 610     // last two iloads in a pair.  Comparing against fast_iload means that
 611     // the next bytecode is neither an iload or a caload, and therefore
 612     // an iload pair.
 613     __ cmp(next_bytecode, Bytecodes::_iload);
 614     __ b(done, eq);
 615 
 616     __ cmp(next_bytecode, Bytecodes::_fast_iload);
 617     __ mov(target_bytecode, Bytecodes::_fast_iload2);
 618     __ b(rewrite, eq);
 619 
 620     // if _caload, rewrite to fast_icaload
 621     __ cmp(next_bytecode, Bytecodes::_caload);
 622     __ mov(target_bytecode, Bytecodes::_fast_icaload);
 623     __ b(rewrite, eq);
 624 
 625     // rewrite so iload doesn't check again.
 626     __ mov(target_bytecode, Bytecodes::_fast_iload);
 627 
 628     // rewrite
 629     // R2: fast bytecode
 630     __ bind(rewrite);
 631     patch_bytecode(Bytecodes::_iload, target_bytecode, Rtemp, false);
 632     __ bind(done);
 633   }
 634 
 635   // Get the local value into tos
 636   const Register Rlocal_index = R1_tmp;
 637   locals_index(Rlocal_index);
 638   Address local = load_iaddress(Rlocal_index, Rtemp);
 639   __ ldr_s32(R0_tos, local);
 640 }
 641 
 642 
 643 void TemplateTable::fast_iload2() {
 644   transition(vtos, itos);
 645   const Register Rlocal_index = R1_tmp;
 646 
 647   locals_index(Rlocal_index);
 648   Address local = load_iaddress(Rlocal_index, Rtemp);
 649   __ ldr_s32(R0_tos, local);
 650   __ push(itos);
 651 
 652   locals_index(Rlocal_index, 3);
 653   local = load_iaddress(Rlocal_index, Rtemp);
 654   __ ldr_s32(R0_tos, local);
 655 }
 656 
 657 void TemplateTable::fast_iload() {
 658   transition(vtos, itos);
 659   const Register Rlocal_index = R1_tmp;
 660 
 661   locals_index(Rlocal_index);
 662   Address local = load_iaddress(Rlocal_index, Rtemp);
 663   __ ldr_s32(R0_tos, local);
 664 }
 665 
 666 
 667 void TemplateTable::lload() {
 668   transition(vtos, ltos);
 669   const Register Rlocal_index = R2_tmp;
 670 
 671   locals_index(Rlocal_index);
 672   load_category2_local(Rlocal_index, R3_tmp);
 673 }
 674 
 675 
 676 void TemplateTable::fload() {
 677   transition(vtos, ftos);
 678   const Register Rlocal_index = R2_tmp;
 679 
 680   // Get the local value into tos
 681   locals_index(Rlocal_index);
 682   Address local = load_faddress(Rlocal_index, Rtemp);
 683 #ifdef __SOFTFP__
 684   __ ldr(R0_tos, local);
 685 #else
 686   __ ldr_float(S0_tos, local);
 687 #endif // __SOFTFP__
 688 }
 689 
 690 
 691 void TemplateTable::dload() {
 692   transition(vtos, dtos);
 693   const Register Rlocal_index = R2_tmp;
 694 
 695   locals_index(Rlocal_index);
 696 
 697 #ifdef __SOFTFP__
 698   load_category2_local(Rlocal_index, R3_tmp);
 699 #else
 700   __ ldr_double(D0_tos, load_daddress(Rlocal_index, Rtemp));
 701 #endif // __SOFTFP__
 702 }
 703 
 704 
 705 void TemplateTable::aload() {
 706   transition(vtos, atos);
 707   const Register Rlocal_index = R1_tmp;
 708 
 709   locals_index(Rlocal_index);
 710   Address local = load_aaddress(Rlocal_index, Rtemp);
 711   __ ldr(R0_tos, local);
 712 }
 713 
 714 
 715 void TemplateTable::locals_index_wide(Register reg) {
 716   assert_different_registers(reg, Rtemp);
 717   __ ldrb(Rtemp, at_bcp(2));
 718   __ ldrb(reg, at_bcp(3));
 719   __ orr(reg, reg, AsmOperand(Rtemp, lsl, 8));
 720 }
 721 
 722 
 723 void TemplateTable::wide_iload() {
 724   transition(vtos, itos);
 725   const Register Rlocal_index = R2_tmp;
 726 
 727   locals_index_wide(Rlocal_index);
 728   Address local = load_iaddress(Rlocal_index, Rtemp);
 729   __ ldr_s32(R0_tos, local);
 730 }
 731 
 732 
 733 void TemplateTable::wide_lload() {
 734   transition(vtos, ltos);
 735   const Register Rlocal_index = R2_tmp;
 736   const Register Rlocal_base = R3_tmp;
 737 
 738   locals_index_wide(Rlocal_index);
 739   load_category2_local(Rlocal_index, R3_tmp);
 740 }
 741 
 742 
 743 void TemplateTable::wide_fload() {
 744   transition(vtos, ftos);
 745   const Register Rlocal_index = R2_tmp;
 746 
 747   locals_index_wide(Rlocal_index);
 748   Address local = load_faddress(Rlocal_index, Rtemp);
 749 #ifdef __SOFTFP__
 750   __ ldr(R0_tos, local);
 751 #else
 752   __ ldr_float(S0_tos, local);
 753 #endif // __SOFTFP__
 754 }
 755 
 756 
 757 void TemplateTable::wide_dload() {
 758   transition(vtos, dtos);
 759   const Register Rlocal_index = R2_tmp;
 760 
 761   locals_index_wide(Rlocal_index);
 762 #ifdef __SOFTFP__
 763   load_category2_local(Rlocal_index, R3_tmp);
 764 #else
 765   __ ldr_double(D0_tos, load_daddress(Rlocal_index, Rtemp));
 766 #endif // __SOFTFP__
 767 }
 768 
 769 
 770 void TemplateTable::wide_aload() {
 771   transition(vtos, atos);
 772   const Register Rlocal_index = R2_tmp;
 773 
 774   locals_index_wide(Rlocal_index);
 775   Address local = load_aaddress(Rlocal_index, Rtemp);
 776   __ ldr(R0_tos, local);
 777 }
 778 
 779 void TemplateTable::index_check(Register array, Register index) {
 780   // Pop ptr into array
 781   __ pop_ptr(array);
 782   index_check_without_pop(array, index);
 783 }
 784 
 785 void TemplateTable::index_check_without_pop(Register array, Register index) {
 786   assert_different_registers(array, index, Rtemp);
 787   // check array
 788   __ null_check(array, Rtemp, arrayOopDesc::length_offset_in_bytes());
 789   // check index
 790   __ ldr_s32(Rtemp, Address(array, arrayOopDesc::length_offset_in_bytes()));
 791   __ cmp_32(index, Rtemp);
 792   if (index != R4_ArrayIndexOutOfBounds_index) {
 793     // convention with generate_ArrayIndexOutOfBounds_handler()
 794     __ mov(R4_ArrayIndexOutOfBounds_index, index, hs);
 795   }
 796   __ b(Interpreter::_throw_ArrayIndexOutOfBoundsException_entry, hs);
 797 }
 798 
 799 
 800 void TemplateTable::iaload() {
 801   transition(itos, itos);
 802   const Register Rarray = R1_tmp;
 803   const Register Rindex = R0_tos;
 804 
 805   index_check(Rarray, Rindex);
 806   __ ldr_s32(R0_tos, get_array_elem_addr(T_INT, Rarray, Rindex, Rtemp));
 807 }
 808 
 809 
 810 void TemplateTable::laload() {
 811   transition(itos, ltos);
 812   const Register Rarray = R1_tmp;
 813   const Register Rindex = R0_tos;
 814 
 815   index_check(Rarray, Rindex);
 816 
 817 #ifdef AARCH64
 818   __ ldr(R0_tos, get_array_elem_addr(T_LONG, Rarray, Rindex, Rtemp));
 819 #else
 820   __ add(Rtemp, Rarray, AsmOperand(Rindex, lsl, LogBytesPerLong));
 821   __ add(Rtemp, Rtemp, arrayOopDesc::base_offset_in_bytes(T_LONG));
 822   __ ldmia(Rtemp, RegisterSet(R0_tos_lo, R1_tos_hi));
 823 #endif // AARCH64
 824 }
 825 
 826 
 827 void TemplateTable::faload() {
 828   transition(itos, ftos);
 829   const Register Rarray = R1_tmp;
 830   const Register Rindex = R0_tos;
 831 
 832   index_check(Rarray, Rindex);
 833 
 834   Address addr = get_array_elem_addr(T_FLOAT, Rarray, Rindex, Rtemp);
 835 #ifdef __SOFTFP__
 836   __ ldr(R0_tos, addr);
 837 #else
 838   __ ldr_float(S0_tos, addr);
 839 #endif // __SOFTFP__
 840 }
 841 
 842 
 843 void TemplateTable::daload() {
 844   transition(itos, dtos);
 845   const Register Rarray = R1_tmp;
 846   const Register Rindex = R0_tos;
 847 
 848   index_check(Rarray, Rindex);
 849 
 850 #ifdef __SOFTFP__
 851   __ add(Rtemp, Rarray, AsmOperand(Rindex, lsl, LogBytesPerLong));
 852   __ add(Rtemp, Rtemp, arrayOopDesc::base_offset_in_bytes(T_DOUBLE));
 853   __ ldmia(Rtemp, RegisterSet(R0_tos_lo, R1_tos_hi));
 854 #else
 855   __ ldr_double(D0_tos, get_array_elem_addr(T_DOUBLE, Rarray, Rindex, Rtemp));
 856 #endif // __SOFTFP__
 857 }
 858 
 859 
 860 void TemplateTable::aaload() {
 861   transition(itos, atos);
 862   const Register Rarray = R1_tmp;
 863   const Register Rindex = R0_tos;
 864 
 865   index_check(Rarray, Rindex);
 866   __ load_heap_oop(R0_tos, get_array_elem_addr(T_OBJECT, Rarray, Rindex, Rtemp));
 867 }
 868 
 869 
 870 void TemplateTable::baload() {
 871   transition(itos, itos);
 872   const Register Rarray = R1_tmp;
 873   const Register Rindex = R0_tos;
 874 
 875   index_check(Rarray, Rindex);
 876   __ ldrsb(R0_tos, get_array_elem_addr(T_BYTE, Rarray, Rindex, Rtemp));
 877 }
 878 
 879 
 880 void TemplateTable::caload() {
 881   transition(itos, itos);
 882   const Register Rarray = R1_tmp;
 883   const Register Rindex = R0_tos;
 884 
 885   index_check(Rarray, Rindex);
 886   __ ldrh(R0_tos, get_array_elem_addr(T_CHAR, Rarray, Rindex, Rtemp));
 887 }
 888 
 889 
 890 // iload followed by caload frequent pair
 891 void TemplateTable::fast_icaload() {
 892   transition(vtos, itos);
 893   const Register Rlocal_index = R1_tmp;
 894   const Register Rarray = R1_tmp;
 895   const Register Rindex = R4_tmp; // index_check prefers index on R4
 896   assert_different_registers(Rlocal_index, Rindex);
 897   assert_different_registers(Rarray, Rindex);
 898 
 899   // load index out of locals
 900   locals_index(Rlocal_index);
 901   Address local = load_iaddress(Rlocal_index, Rtemp);
 902   __ ldr_s32(Rindex, local);
 903 
 904   // get array element
 905   index_check(Rarray, Rindex);
 906   __ ldrh(R0_tos, get_array_elem_addr(T_CHAR, Rarray, Rindex, Rtemp));
 907 }
 908 
 909 
 910 void TemplateTable::saload() {
 911   transition(itos, itos);
 912   const Register Rarray = R1_tmp;
 913   const Register Rindex = R0_tos;
 914 
 915   index_check(Rarray, Rindex);
 916   __ ldrsh(R0_tos, get_array_elem_addr(T_SHORT, Rarray, Rindex, Rtemp));
 917 }
 918 
 919 
 920 void TemplateTable::iload(int n) {
 921   transition(vtos, itos);
 922   __ ldr_s32(R0_tos, iaddress(n));
 923 }
 924 
 925 
 926 void TemplateTable::lload(int n) {
 927   transition(vtos, ltos);
 928 #ifdef AARCH64
 929   __ ldr(R0_tos, laddress(n));
 930 #else
 931   __ ldr(R0_tos_lo, laddress(n));
 932   __ ldr(R1_tos_hi, haddress(n));
 933 #endif // AARCH64
 934 }
 935 
 936 
 937 void TemplateTable::fload(int n) {
 938   transition(vtos, ftos);
 939 #ifdef __SOFTFP__
 940   __ ldr(R0_tos, faddress(n));
 941 #else
 942   __ ldr_float(S0_tos, faddress(n));
 943 #endif // __SOFTFP__
 944 }
 945 
 946 
 947 void TemplateTable::dload(int n) {
 948   transition(vtos, dtos);
 949 #ifdef __SOFTFP__
 950   __ ldr(R0_tos_lo, laddress(n));
 951   __ ldr(R1_tos_hi, haddress(n));
 952 #else
 953   __ ldr_double(D0_tos, daddress(n));
 954 #endif // __SOFTFP__
 955 }
 956 
 957 
 958 void TemplateTable::aload(int n) {
 959   transition(vtos, atos);
 960   __ ldr(R0_tos, aaddress(n));
 961 }
 962 
 963 void TemplateTable::aload_0() {
 964   aload_0_internal();
 965 }
 966 
 967 void TemplateTable::nofast_aload_0() {
 968   aload_0_internal(may_not_rewrite);
 969 }
 970 
 971 void TemplateTable::aload_0_internal(RewriteControl rc) {
 972   transition(vtos, atos);
 973   // According to bytecode histograms, the pairs:
 974   //
 975   // _aload_0, _fast_igetfield
 976   // _aload_0, _fast_agetfield
 977   // _aload_0, _fast_fgetfield
 978   //
 979   // occur frequently. If RewriteFrequentPairs is set, the (slow) _aload_0
 980   // bytecode checks if the next bytecode is either _fast_igetfield,
 981   // _fast_agetfield or _fast_fgetfield and then rewrites the
 982   // current bytecode into a pair bytecode; otherwise it rewrites the current
 983   // bytecode into _fast_aload_0 that doesn't do the pair check anymore.
 984   //
 985   // Note: If the next bytecode is _getfield, the rewrite must be delayed,
 986   //       otherwise we may miss an opportunity for a pair.
 987   //
 988   // Also rewrite frequent pairs
 989   //   aload_0, aload_1
 990   //   aload_0, iload_1
 991   // These bytecodes with a small amount of code are most profitable to rewrite
 992   if ((rc == may_rewrite) && __ rewrite_frequent_pairs()) {
 993     Label rewrite, done;
 994     const Register next_bytecode = R1_tmp;
 995     const Register target_bytecode = R2_tmp;
 996 
 997     // get next byte
 998     __ ldrb(next_bytecode, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 999 
1000     // if _getfield then wait with rewrite
1001     __ cmp(next_bytecode, Bytecodes::_getfield);
1002     __ b(done, eq);
1003 
1004     // if _igetfield then rewrite to _fast_iaccess_0
1005     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
1006     __ cmp(next_bytecode, Bytecodes::_fast_igetfield);
1007     __ mov(target_bytecode, Bytecodes::_fast_iaccess_0);
1008     __ b(rewrite, eq);
1009 
1010     // if _agetfield then rewrite to _fast_aaccess_0
1011     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
1012     __ cmp(next_bytecode, Bytecodes::_fast_agetfield);
1013     __ mov(target_bytecode, Bytecodes::_fast_aaccess_0);
1014     __ b(rewrite, eq);
1015 
1016     // if _fgetfield then rewrite to _fast_faccess_0, else rewrite to _fast_aload0
1017     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
1018     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
1019 
1020     __ cmp(next_bytecode, Bytecodes::_fast_fgetfield);
1021 #ifdef AARCH64
1022     __ mov(Rtemp, Bytecodes::_fast_faccess_0);
1023     __ mov(target_bytecode, Bytecodes::_fast_aload_0);
1024     __ mov(target_bytecode, Rtemp, eq);
1025 #else
1026     __ mov(target_bytecode, Bytecodes::_fast_faccess_0, eq);
1027     __ mov(target_bytecode, Bytecodes::_fast_aload_0, ne);
1028 #endif // AARCH64
1029 
1030     // rewrite
1031     __ bind(rewrite);
1032     patch_bytecode(Bytecodes::_aload_0, target_bytecode, Rtemp, false);
1033 
1034     __ bind(done);
1035   }
1036 
1037   aload(0);
1038 }
1039 
1040 void TemplateTable::istore() {
1041   transition(itos, vtos);
1042   const Register Rlocal_index = R2_tmp;
1043 
1044   locals_index(Rlocal_index);
1045   Address local = load_iaddress(Rlocal_index, Rtemp);
1046   __ str_32(R0_tos, local);
1047 }
1048 
1049 
1050 void TemplateTable::lstore() {
1051   transition(ltos, vtos);
1052   const Register Rlocal_index = R2_tmp;
1053 
1054   locals_index(Rlocal_index);
1055   store_category2_local(Rlocal_index, R3_tmp);
1056 }
1057 
1058 
1059 void TemplateTable::fstore() {
1060   transition(ftos, vtos);
1061   const Register Rlocal_index = R2_tmp;
1062 
1063   locals_index(Rlocal_index);
1064   Address local = load_faddress(Rlocal_index, Rtemp);
1065 #ifdef __SOFTFP__
1066   __ str(R0_tos, local);
1067 #else
1068   __ str_float(S0_tos, local);
1069 #endif // __SOFTFP__
1070 }
1071 
1072 
1073 void TemplateTable::dstore() {
1074   transition(dtos, vtos);
1075   const Register Rlocal_index = R2_tmp;
1076 
1077   locals_index(Rlocal_index);
1078 
1079 #ifdef __SOFTFP__
1080   store_category2_local(Rlocal_index, R3_tmp);
1081 #else
1082   __ str_double(D0_tos, load_daddress(Rlocal_index, Rtemp));
1083 #endif // __SOFTFP__
1084 }
1085 
1086 
1087 void TemplateTable::astore() {
1088   transition(vtos, vtos);
1089   const Register Rlocal_index = R1_tmp;
1090 
1091   __ pop_ptr(R0_tos);
1092   locals_index(Rlocal_index);
1093   Address local = load_aaddress(Rlocal_index, Rtemp);
1094   __ str(R0_tos, local);
1095 }
1096 
1097 
1098 void TemplateTable::wide_istore() {
1099   transition(vtos, vtos);
1100   const Register Rlocal_index = R2_tmp;
1101 
1102   __ pop_i(R0_tos);
1103   locals_index_wide(Rlocal_index);
1104   Address local = load_iaddress(Rlocal_index, Rtemp);
1105   __ str_32(R0_tos, local);
1106 }
1107 
1108 
1109 void TemplateTable::wide_lstore() {
1110   transition(vtos, vtos);
1111   const Register Rlocal_index = R2_tmp;
1112   const Register Rlocal_base = R3_tmp;
1113 
1114 #ifdef AARCH64
1115   __ pop_l(R0_tos);
1116 #else
1117   __ pop_l(R0_tos_lo, R1_tos_hi);
1118 #endif // AARCH64
1119 
1120   locals_index_wide(Rlocal_index);
1121   store_category2_local(Rlocal_index, R3_tmp);
1122 }
1123 
1124 
1125 void TemplateTable::wide_fstore() {
1126   wide_istore();
1127 }
1128 
1129 
1130 void TemplateTable::wide_dstore() {
1131   wide_lstore();
1132 }
1133 
1134 
1135 void TemplateTable::wide_astore() {
1136   transition(vtos, vtos);
1137   const Register Rlocal_index = R2_tmp;
1138 
1139   __ pop_ptr(R0_tos);
1140   locals_index_wide(Rlocal_index);
1141   Address local = load_aaddress(Rlocal_index, Rtemp);
1142   __ str(R0_tos, local);
1143 }
1144 
1145 
1146 void TemplateTable::iastore() {
1147   transition(itos, vtos);
1148   const Register Rindex = R4_tmp; // index_check prefers index in R4
1149   const Register Rarray = R3_tmp;
1150   // R0_tos: value
1151 
1152   __ pop_i(Rindex);
1153   index_check(Rarray, Rindex);
1154   __ str_32(R0_tos, get_array_elem_addr(T_INT, Rarray, Rindex, Rtemp));
1155 }
1156 
1157 
1158 void TemplateTable::lastore() {
1159   transition(ltos, vtos);
1160   const Register Rindex = R4_tmp; // index_check prefers index in R4
1161   const Register Rarray = R3_tmp;
1162   // R0_tos_lo:R1_tos_hi: value
1163 
1164   __ pop_i(Rindex);
1165   index_check(Rarray, Rindex);
1166 
1167 #ifdef AARCH64
1168   __ str(R0_tos, get_array_elem_addr(T_LONG, Rarray, Rindex, Rtemp));
1169 #else
1170   __ add(Rtemp, Rarray, AsmOperand(Rindex, lsl, LogBytesPerLong));
1171   __ add(Rtemp, Rtemp, arrayOopDesc::base_offset_in_bytes(T_LONG));
1172   __ stmia(Rtemp, RegisterSet(R0_tos_lo, R1_tos_hi));
1173 #endif // AARCH64
1174 }
1175 
1176 
1177 void TemplateTable::fastore() {
1178   transition(ftos, vtos);
1179   const Register Rindex = R4_tmp; // index_check prefers index in R4
1180   const Register Rarray = R3_tmp;
1181   // S0_tos/R0_tos: value
1182 
1183   __ pop_i(Rindex);
1184   index_check(Rarray, Rindex);
1185   Address addr = get_array_elem_addr(T_FLOAT, Rarray, Rindex, Rtemp);
1186 
1187 #ifdef __SOFTFP__
1188   __ str(R0_tos, addr);
1189 #else
1190   __ str_float(S0_tos, addr);
1191 #endif // __SOFTFP__
1192 }
1193 
1194 
1195 void TemplateTable::dastore() {
1196   transition(dtos, vtos);
1197   const Register Rindex = R4_tmp; // index_check prefers index in R4
1198   const Register Rarray = R3_tmp;
1199   // D0_tos / R0_tos_lo:R1_to_hi: value
1200 
1201   __ pop_i(Rindex);
1202   index_check(Rarray, Rindex);
1203 
1204 #ifdef __SOFTFP__
1205   __ add(Rtemp, Rarray, AsmOperand(Rindex, lsl, LogBytesPerLong));
1206   __ add(Rtemp, Rtemp, arrayOopDesc::base_offset_in_bytes(T_DOUBLE));
1207   __ stmia(Rtemp, RegisterSet(R0_tos_lo, R1_tos_hi));
1208 #else
1209   __ str_double(D0_tos, get_array_elem_addr(T_DOUBLE, Rarray, Rindex, Rtemp));
1210 #endif // __SOFTFP__
1211 }
1212 
1213 
1214 void TemplateTable::aastore() {
1215   transition(vtos, vtos);
1216   Label is_null, throw_array_store, done;
1217 
1218   const Register Raddr_1   = R1_tmp;
1219   const Register Rvalue_2  = R2_tmp;
1220   const Register Rarray_3  = R3_tmp;
1221   const Register Rindex_4  = R4_tmp;   // preferred by index_check_without_pop()
1222   const Register Rsub_5    = R5_tmp;
1223   const Register Rsuper_LR = LR_tmp;
1224 
1225   // stack: ..., array, index, value
1226   __ ldr(Rvalue_2, at_tos());     // Value
1227   __ ldr_s32(Rindex_4, at_tos_p1());  // Index
1228   __ ldr(Rarray_3, at_tos_p2());  // Array
1229 
1230   index_check_without_pop(Rarray_3, Rindex_4);
1231 
1232   // Compute the array base
1233   __ add(Raddr_1, Rarray_3, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1234 
1235   // do array store check - check for NULL value first
1236   __ cbz(Rvalue_2, is_null);
1237 
1238   // Load subklass
1239   __ load_klass(Rsub_5, Rvalue_2);
1240   // Load superklass
1241   __ load_klass(Rtemp, Rarray_3);
1242   __ ldr(Rsuper_LR, Address(Rtemp, ObjArrayKlass::element_klass_offset()));
1243 
1244   __ gen_subtype_check(Rsub_5, Rsuper_LR, throw_array_store, R0_tmp, R3_tmp);
1245   // Come here on success
1246 
1247   // Store value
1248   __ add(Raddr_1, Raddr_1, AsmOperand(Rindex_4, lsl, LogBytesPerHeapOop));
1249 
1250   // Now store using the appropriate barrier
1251   do_oop_store(_masm, Raddr_1, Rvalue_2, Rtemp, R0_tmp, R3_tmp, _bs->kind(), true, false);
1252   __ b(done);
1253 
1254   __ bind(throw_array_store);
1255 
1256   // Come here on failure of subtype check
1257   __ profile_typecheck_failed(R0_tmp);
1258 
1259   // object is at TOS
1260   __ b(Interpreter::_throw_ArrayStoreException_entry);
1261 
1262   // Have a NULL in Rvalue_2, store NULL at array[index].
1263   __ bind(is_null);
1264   __ profile_null_seen(R0_tmp);
1265 
1266   // Store a NULL
1267   do_oop_store(_masm, Address::indexed_oop(Raddr_1, Rindex_4), Rvalue_2, Rtemp, R0_tmp, R3_tmp, _bs->kind(), true, true);
1268 
1269   // Pop stack arguments
1270   __ bind(done);
1271   __ add(Rstack_top, Rstack_top, 3 * Interpreter::stackElementSize);
1272 }
1273 
1274 
1275 void TemplateTable::bastore() {
1276   transition(itos, vtos);
1277   const Register Rindex = R4_tmp; // index_check prefers index in R4
1278   const Register Rarray = R3_tmp;
1279   // R0_tos: value
1280 
1281   __ pop_i(Rindex);
1282   index_check(Rarray, Rindex);
1283 
1284   // Need to check whether array is boolean or byte
1285   // since both types share the bastore bytecode.
1286   __ load_klass(Rtemp, Rarray);
1287   __ ldr_u32(Rtemp, Address(Rtemp, Klass::layout_helper_offset()));
1288   Label L_skip;
1289   __ tst(Rtemp, Klass::layout_helper_boolean_diffbit());
1290   __ b(L_skip, eq);
1291   __ and_32(R0_tos, R0_tos, 1); // if it is a T_BOOLEAN array, mask the stored value to 0/1
1292   __ bind(L_skip);
1293   __ strb(R0_tos, get_array_elem_addr(T_BYTE, Rarray, Rindex, Rtemp));
1294 }
1295 
1296 
1297 void TemplateTable::castore() {
1298   transition(itos, vtos);
1299   const Register Rindex = R4_tmp; // index_check prefers index in R4
1300   const Register Rarray = R3_tmp;
1301   // R0_tos: value
1302 
1303   __ pop_i(Rindex);
1304   index_check(Rarray, Rindex);
1305 
1306   __ strh(R0_tos, get_array_elem_addr(T_CHAR, Rarray, Rindex, Rtemp));
1307 }
1308 
1309 
1310 void TemplateTable::sastore() {
1311   assert(arrayOopDesc::base_offset_in_bytes(T_CHAR) ==
1312            arrayOopDesc::base_offset_in_bytes(T_SHORT),
1313          "base offsets for char and short should be equal");
1314   castore();
1315 }
1316 
1317 
1318 void TemplateTable::istore(int n) {
1319   transition(itos, vtos);
1320   __ str_32(R0_tos, iaddress(n));
1321 }
1322 
1323 
1324 void TemplateTable::lstore(int n) {
1325   transition(ltos, vtos);
1326 #ifdef AARCH64
1327   __ str(R0_tos, laddress(n));
1328 #else
1329   __ str(R0_tos_lo, laddress(n));
1330   __ str(R1_tos_hi, haddress(n));
1331 #endif // AARCH64
1332 }
1333 
1334 
1335 void TemplateTable::fstore(int n) {
1336   transition(ftos, vtos);
1337 #ifdef __SOFTFP__
1338   __ str(R0_tos, faddress(n));
1339 #else
1340   __ str_float(S0_tos, faddress(n));
1341 #endif // __SOFTFP__
1342 }
1343 
1344 
1345 void TemplateTable::dstore(int n) {
1346   transition(dtos, vtos);
1347 #ifdef __SOFTFP__
1348   __ str(R0_tos_lo, laddress(n));
1349   __ str(R1_tos_hi, haddress(n));
1350 #else
1351   __ str_double(D0_tos, daddress(n));
1352 #endif // __SOFTFP__
1353 }
1354 
1355 
1356 void TemplateTable::astore(int n) {
1357   transition(vtos, vtos);
1358   __ pop_ptr(R0_tos);
1359   __ str(R0_tos, aaddress(n));
1360 }
1361 
1362 
1363 void TemplateTable::pop() {
1364   transition(vtos, vtos);
1365   __ add(Rstack_top, Rstack_top, Interpreter::stackElementSize);
1366 }
1367 
1368 
1369 void TemplateTable::pop2() {
1370   transition(vtos, vtos);
1371   __ add(Rstack_top, Rstack_top, 2*Interpreter::stackElementSize);
1372 }
1373 
1374 
1375 void TemplateTable::dup() {
1376   transition(vtos, vtos);
1377   // stack: ..., a
1378   __ load_ptr(0, R0_tmp);
1379   __ push_ptr(R0_tmp);
1380   // stack: ..., a, a
1381 }
1382 
1383 
1384 void TemplateTable::dup_x1() {
1385   transition(vtos, vtos);
1386   // stack: ..., a, b
1387   __ load_ptr(0, R0_tmp);  // load b
1388   __ load_ptr(1, R2_tmp);  // load a
1389   __ store_ptr(1, R0_tmp); // store b
1390   __ store_ptr(0, R2_tmp); // store a
1391   __ push_ptr(R0_tmp);     // push b
1392   // stack: ..., b, a, b
1393 }
1394 
1395 
1396 void TemplateTable::dup_x2() {
1397   transition(vtos, vtos);
1398   // stack: ..., a, b, c
1399   __ load_ptr(0, R0_tmp);   // load c
1400   __ load_ptr(1, R2_tmp);   // load b
1401   __ load_ptr(2, R4_tmp);   // load a
1402 
1403   __ push_ptr(R0_tmp);      // push c
1404 
1405   // stack: ..., a, b, c, c
1406   __ store_ptr(1, R2_tmp);  // store b
1407   __ store_ptr(2, R4_tmp);  // store a
1408   __ store_ptr(3, R0_tmp);  // store c
1409   // stack: ..., c, a, b, c
1410 }
1411 
1412 
1413 void TemplateTable::dup2() {
1414   transition(vtos, vtos);
1415   // stack: ..., a, b
1416   __ load_ptr(1, R0_tmp);  // load a
1417   __ push_ptr(R0_tmp);     // push a
1418   __ load_ptr(1, R0_tmp);  // load b
1419   __ push_ptr(R0_tmp);     // push b
1420   // stack: ..., a, b, a, b
1421 }
1422 
1423 
1424 void TemplateTable::dup2_x1() {
1425   transition(vtos, vtos);
1426 
1427   // stack: ..., a, b, c
1428   __ load_ptr(0, R4_tmp);  // load c
1429   __ load_ptr(1, R2_tmp);  // load b
1430   __ load_ptr(2, R0_tmp);  // load a
1431 
1432   __ push_ptr(R2_tmp);     // push b
1433   __ push_ptr(R4_tmp);     // push c
1434 
1435   // stack: ..., a, b, c, b, c
1436 
1437   __ store_ptr(2, R0_tmp);  // store a
1438   __ store_ptr(3, R4_tmp);  // store c
1439   __ store_ptr(4, R2_tmp);  // store b
1440 
1441   // stack: ..., b, c, a, b, c
1442 }
1443 
1444 
1445 void TemplateTable::dup2_x2() {
1446   transition(vtos, vtos);
1447   // stack: ..., a, b, c, d
1448   __ load_ptr(0, R0_tmp);  // load d
1449   __ load_ptr(1, R2_tmp);  // load c
1450   __ push_ptr(R2_tmp);     // push c
1451   __ push_ptr(R0_tmp);     // push d
1452   // stack: ..., a, b, c, d, c, d
1453   __ load_ptr(4, R4_tmp);  // load b
1454   __ store_ptr(4, R0_tmp); // store d in b
1455   __ store_ptr(2, R4_tmp); // store b in d
1456   // stack: ..., a, d, c, b, c, d
1457   __ load_ptr(5, R4_tmp);  // load a
1458   __ store_ptr(5, R2_tmp); // store c in a
1459   __ store_ptr(3, R4_tmp); // store a in c
1460   // stack: ..., c, d, a, b, c, d
1461 }
1462 
1463 
1464 void TemplateTable::swap() {
1465   transition(vtos, vtos);
1466   // stack: ..., a, b
1467   __ load_ptr(1, R0_tmp);  // load a
1468   __ load_ptr(0, R2_tmp);  // load b
1469   __ store_ptr(0, R0_tmp); // store a in b
1470   __ store_ptr(1, R2_tmp); // store b in a
1471   // stack: ..., b, a
1472 }
1473 
1474 
1475 void TemplateTable::iop2(Operation op) {
1476   transition(itos, itos);
1477   const Register arg1 = R1_tmp;
1478   const Register arg2 = R0_tos;
1479 
1480   __ pop_i(arg1);
1481   switch (op) {
1482     case add  : __ add_32 (R0_tos, arg1, arg2); break;
1483     case sub  : __ sub_32 (R0_tos, arg1, arg2); break;
1484     case mul  : __ mul_32 (R0_tos, arg1, arg2); break;
1485     case _and : __ and_32 (R0_tos, arg1, arg2); break;
1486     case _or  : __ orr_32 (R0_tos, arg1, arg2); break;
1487     case _xor : __ eor_32 (R0_tos, arg1, arg2); break;
1488 #ifdef AARCH64
1489     case shl  : __ lslv_w (R0_tos, arg1, arg2); break;
1490     case shr  : __ asrv_w (R0_tos, arg1, arg2); break;
1491     case ushr : __ lsrv_w (R0_tos, arg1, arg2); break;
1492 #else
1493     case shl  : __ andr(arg2, arg2, 0x1f); __ mov (R0_tos, AsmOperand(arg1, lsl, arg2)); break;
1494     case shr  : __ andr(arg2, arg2, 0x1f); __ mov (R0_tos, AsmOperand(arg1, asr, arg2)); break;
1495     case ushr : __ andr(arg2, arg2, 0x1f); __ mov (R0_tos, AsmOperand(arg1, lsr, arg2)); break;
1496 #endif // AARCH64
1497     default   : ShouldNotReachHere();
1498   }
1499 }
1500 
1501 
1502 void TemplateTable::lop2(Operation op) {
1503   transition(ltos, ltos);
1504 #ifdef AARCH64
1505   const Register arg1 = R1_tmp;
1506   const Register arg2 = R0_tos;
1507 
1508   __ pop_l(arg1);
1509   switch (op) {
1510     case add  : __ add (R0_tos, arg1, arg2); break;
1511     case sub  : __ sub (R0_tos, arg1, arg2); break;
1512     case _and : __ andr(R0_tos, arg1, arg2); break;
1513     case _or  : __ orr (R0_tos, arg1, arg2); break;
1514     case _xor : __ eor (R0_tos, arg1, arg2); break;
1515     default   : ShouldNotReachHere();
1516   }
1517 #else
1518   const Register arg1_lo = R2_tmp;
1519   const Register arg1_hi = R3_tmp;
1520   const Register arg2_lo = R0_tos_lo;
1521   const Register arg2_hi = R1_tos_hi;
1522 
1523   __ pop_l(arg1_lo, arg1_hi);
1524   switch (op) {
1525     case add : __ adds(R0_tos_lo, arg1_lo, arg2_lo); __ adc (R1_tos_hi, arg1_hi, arg2_hi); break;
1526     case sub : __ subs(R0_tos_lo, arg1_lo, arg2_lo); __ sbc (R1_tos_hi, arg1_hi, arg2_hi); break;
1527     case _and: __ andr(R0_tos_lo, arg1_lo, arg2_lo); __ andr(R1_tos_hi, arg1_hi, arg2_hi); break;
1528     case _or : __ orr (R0_tos_lo, arg1_lo, arg2_lo); __ orr (R1_tos_hi, arg1_hi, arg2_hi); break;
1529     case _xor: __ eor (R0_tos_lo, arg1_lo, arg2_lo); __ eor (R1_tos_hi, arg1_hi, arg2_hi); break;
1530     default : ShouldNotReachHere();
1531   }
1532 #endif // AARCH64
1533 }
1534 
1535 
1536 void TemplateTable::idiv() {
1537   transition(itos, itos);
1538 #ifdef AARCH64
1539   const Register divisor = R0_tos;
1540   const Register dividend = R1_tmp;
1541 
1542   __ cbz_w(divisor, Interpreter::_throw_ArithmeticException_entry);
1543   __ pop_i(dividend);
1544   __ sdiv_w(R0_tos, dividend, divisor);
1545 #else
1546   __ mov(R2, R0_tos);
1547   __ pop_i(R0);
1548   // R0 - dividend
1549   // R2 - divisor
1550   __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::none);
1551   // R1 - result
1552   __ mov(R0_tos, R1);
1553 #endif // AARCH64
1554 }
1555 
1556 
1557 void TemplateTable::irem() {
1558   transition(itos, itos);
1559 #ifdef AARCH64
1560   const Register divisor = R0_tos;
1561   const Register dividend = R1_tmp;
1562   const Register quotient = R2_tmp;
1563 
1564   __ cbz_w(divisor, Interpreter::_throw_ArithmeticException_entry);
1565   __ pop_i(dividend);
1566   __ sdiv_w(quotient, dividend, divisor);
1567   __ msub_w(R0_tos, divisor, quotient, dividend);
1568 #else
1569   __ mov(R2, R0_tos);
1570   __ pop_i(R0);
1571   // R0 - dividend
1572   // R2 - divisor
1573   __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::none);
1574   // R0 - remainder
1575 #endif // AARCH64
1576 }
1577 
1578 
1579 void TemplateTable::lmul() {
1580   transition(ltos, ltos);
1581 #ifdef AARCH64
1582   const Register arg1 = R0_tos;
1583   const Register arg2 = R1_tmp;
1584 
1585   __ pop_l(arg2);
1586   __ mul(R0_tos, arg1, arg2);
1587 #else
1588   const Register arg1_lo = R0_tos_lo;
1589   const Register arg1_hi = R1_tos_hi;
1590   const Register arg2_lo = R2_tmp;
1591   const Register arg2_hi = R3_tmp;
1592 
1593   __ pop_l(arg2_lo, arg2_hi);
1594 
1595   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::lmul), arg1_lo, arg1_hi, arg2_lo, arg2_hi);
1596 #endif // AARCH64
1597 }
1598 
1599 
1600 void TemplateTable::ldiv() {
1601   transition(ltos, ltos);
1602 #ifdef AARCH64
1603   const Register divisor = R0_tos;
1604   const Register dividend = R1_tmp;
1605 
1606   __ cbz(divisor, Interpreter::_throw_ArithmeticException_entry);
1607   __ pop_l(dividend);
1608   __ sdiv(R0_tos, dividend, divisor);
1609 #else
1610   const Register x_lo = R2_tmp;
1611   const Register x_hi = R3_tmp;
1612   const Register y_lo = R0_tos_lo;
1613   const Register y_hi = R1_tos_hi;
1614 
1615   __ pop_l(x_lo, x_hi);
1616 
1617   // check if y = 0
1618   __ orrs(Rtemp, y_lo, y_hi);
1619   __ call(Interpreter::_throw_ArithmeticException_entry, relocInfo::none, eq);
1620   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::ldiv), y_lo, y_hi, x_lo, x_hi);
1621 #endif // AARCH64
1622 }
1623 
1624 
1625 void TemplateTable::lrem() {
1626   transition(ltos, ltos);
1627 #ifdef AARCH64
1628   const Register divisor = R0_tos;
1629   const Register dividend = R1_tmp;
1630   const Register quotient = R2_tmp;
1631 
1632   __ cbz(divisor, Interpreter::_throw_ArithmeticException_entry);
1633   __ pop_l(dividend);
1634   __ sdiv(quotient, dividend, divisor);
1635   __ msub(R0_tos, divisor, quotient, dividend);
1636 #else
1637   const Register x_lo = R2_tmp;
1638   const Register x_hi = R3_tmp;
1639   const Register y_lo = R0_tos_lo;
1640   const Register y_hi = R1_tos_hi;
1641 
1642   __ pop_l(x_lo, x_hi);
1643 
1644   // check if y = 0
1645   __ orrs(Rtemp, y_lo, y_hi);
1646   __ call(Interpreter::_throw_ArithmeticException_entry, relocInfo::none, eq);
1647   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::lrem), y_lo, y_hi, x_lo, x_hi);
1648 #endif // AARCH64
1649 }
1650 
1651 
1652 void TemplateTable::lshl() {
1653   transition(itos, ltos);
1654 #ifdef AARCH64
1655   const Register val = R1_tmp;
1656   const Register shift_cnt = R0_tos;
1657   __ pop_l(val);
1658   __ lslv(R0_tos, val, shift_cnt);
1659 #else
1660   const Register shift_cnt = R4_tmp;
1661   const Register val_lo = R2_tmp;
1662   const Register val_hi = R3_tmp;
1663 
1664   __ pop_l(val_lo, val_hi);
1665   __ andr(shift_cnt, R0_tos, 63);
1666   __ long_shift(R0_tos_lo, R1_tos_hi, val_lo, val_hi, lsl, shift_cnt);
1667 #endif // AARCH64
1668 }
1669 
1670 
1671 void TemplateTable::lshr() {
1672   transition(itos, ltos);
1673 #ifdef AARCH64
1674   const Register val = R1_tmp;
1675   const Register shift_cnt = R0_tos;
1676   __ pop_l(val);
1677   __ asrv(R0_tos, val, shift_cnt);
1678 #else
1679   const Register shift_cnt = R4_tmp;
1680   const Register val_lo = R2_tmp;
1681   const Register val_hi = R3_tmp;
1682 
1683   __ pop_l(val_lo, val_hi);
1684   __ andr(shift_cnt, R0_tos, 63);
1685   __ long_shift(R0_tos_lo, R1_tos_hi, val_lo, val_hi, asr, shift_cnt);
1686 #endif // AARCH64
1687 }
1688 
1689 
1690 void TemplateTable::lushr() {
1691   transition(itos, ltos);
1692 #ifdef AARCH64
1693   const Register val = R1_tmp;
1694   const Register shift_cnt = R0_tos;
1695   __ pop_l(val);
1696   __ lsrv(R0_tos, val, shift_cnt);
1697 #else
1698   const Register shift_cnt = R4_tmp;
1699   const Register val_lo = R2_tmp;
1700   const Register val_hi = R3_tmp;
1701 
1702   __ pop_l(val_lo, val_hi);
1703   __ andr(shift_cnt, R0_tos, 63);
1704   __ long_shift(R0_tos_lo, R1_tos_hi, val_lo, val_hi, lsr, shift_cnt);
1705 #endif // AARCH64
1706 }
1707 
1708 
1709 void TemplateTable::fop2(Operation op) {
1710   transition(ftos, ftos);
1711 #ifdef __SOFTFP__
1712   __ mov(R1, R0_tos);
1713   __ pop_i(R0);
1714   switch (op) {
1715     case add: __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_fadd_glibc), R0, R1); break;
1716     case sub: __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_fsub_glibc), R0, R1); break;
1717     case mul: __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_fmul), R0, R1); break;
1718     case div: __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_fdiv), R0, R1); break;
1719     case rem: __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem), R0, R1); break;
1720     default : ShouldNotReachHere();
1721   }
1722 #else
1723   const FloatRegister arg1 = S1_tmp;
1724   const FloatRegister arg2 = S0_tos;
1725 
1726   switch (op) {
1727     case add: __ pop_f(arg1); __ add_float(S0_tos, arg1, arg2); break;
1728     case sub: __ pop_f(arg1); __ sub_float(S0_tos, arg1, arg2); break;
1729     case mul: __ pop_f(arg1); __ mul_float(S0_tos, arg1, arg2); break;
1730     case div: __ pop_f(arg1); __ div_float(S0_tos, arg1, arg2); break;
1731     case rem:
1732 #ifndef __ABI_HARD__
1733       __ pop_f(arg1);
1734       __ fmrs(R0, arg1);
1735       __ fmrs(R1, arg2);
1736       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem), R0, R1);
1737       __ fmsr(S0_tos, R0);
1738 #else
1739       __ mov_float(S1_reg, arg2);
1740       __ pop_f(S0);
1741       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1742 #endif // !__ABI_HARD__
1743       break;
1744     default : ShouldNotReachHere();
1745   }
1746 #endif // __SOFTFP__
1747 }
1748 
1749 
1750 void TemplateTable::dop2(Operation op) {
1751   transition(dtos, dtos);
1752 #ifdef __SOFTFP__
1753   __ mov(R2, R0_tos_lo);
1754   __ mov(R3, R1_tos_hi);
1755   __ pop_l(R0, R1);
1756   switch (op) {
1757     // __aeabi_XXXX_glibc: Imported code from glibc soft-fp bundle for calculation accuracy improvement. See CR 6757269.
1758     case add: __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_dadd_glibc), R0, R1, R2, R3); break;
1759     case sub: __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_dsub_glibc), R0, R1, R2, R3); break;
1760     case mul: __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_dmul), R0, R1, R2, R3); break;
1761     case div: __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_ddiv), R0, R1, R2, R3); break;
1762     case rem: __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem), R0, R1, R2, R3); break;
1763     default : ShouldNotReachHere();
1764   }
1765 #else
1766   const FloatRegister arg1 = D1_tmp;
1767   const FloatRegister arg2 = D0_tos;
1768 
1769   switch (op) {
1770     case add: __ pop_d(arg1); __ add_double(D0_tos, arg1, arg2); break;
1771     case sub: __ pop_d(arg1); __ sub_double(D0_tos, arg1, arg2); break;
1772     case mul: __ pop_d(arg1); __ mul_double(D0_tos, arg1, arg2); break;
1773     case div: __ pop_d(arg1); __ div_double(D0_tos, arg1, arg2); break;
1774     case rem:
1775 #ifndef __ABI_HARD__
1776       __ pop_d(arg1);
1777       __ fmrrd(R0, R1, arg1);
1778       __ fmrrd(R2, R3, arg2);
1779       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem), R0, R1, R2, R3);
1780       __ fmdrr(D0_tos, R0, R1);
1781 #else
1782       __ mov_double(D1, arg2);
1783       __ pop_d(D0);
1784       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1785 #endif // !__ABI_HARD__
1786       break;
1787     default : ShouldNotReachHere();
1788   }
1789 #endif // __SOFTFP__
1790 }
1791 
1792 
1793 void TemplateTable::ineg() {
1794   transition(itos, itos);
1795   __ neg_32(R0_tos, R0_tos);
1796 }
1797 
1798 
1799 void TemplateTable::lneg() {
1800   transition(ltos, ltos);
1801 #ifdef AARCH64
1802   __ neg(R0_tos, R0_tos);
1803 #else
1804   __ rsbs(R0_tos_lo, R0_tos_lo, 0);
1805   __ rsc (R1_tos_hi, R1_tos_hi, 0);
1806 #endif // AARCH64
1807 }
1808 
1809 
1810 void TemplateTable::fneg() {
1811   transition(ftos, ftos);
1812 #ifdef __SOFTFP__
1813   // Invert sign bit
1814   const int sign_mask = 0x80000000;
1815   __ eor(R0_tos, R0_tos, sign_mask);
1816 #else
1817   __ neg_float(S0_tos, S0_tos);
1818 #endif // __SOFTFP__
1819 }
1820 
1821 
1822 void TemplateTable::dneg() {
1823   transition(dtos, dtos);
1824 #ifdef __SOFTFP__
1825   // Invert sign bit in the high part of the double
1826   const int sign_mask_hi = 0x80000000;
1827   __ eor(R1_tos_hi, R1_tos_hi, sign_mask_hi);
1828 #else
1829   __ neg_double(D0_tos, D0_tos);
1830 #endif // __SOFTFP__
1831 }
1832 
1833 
1834 void TemplateTable::iinc() {
1835   transition(vtos, vtos);
1836   const Register Rconst = R2_tmp;
1837   const Register Rlocal_index = R1_tmp;
1838   const Register Rval = R0_tmp;
1839 
1840   __ ldrsb(Rconst, at_bcp(2));
1841   locals_index(Rlocal_index);
1842   Address local = load_iaddress(Rlocal_index, Rtemp);
1843   __ ldr_s32(Rval, local);
1844   __ add(Rval, Rval, Rconst);
1845   __ str_32(Rval, local);
1846 }
1847 
1848 
1849 void TemplateTable::wide_iinc() {
1850   transition(vtos, vtos);
1851   const Register Rconst = R2_tmp;
1852   const Register Rlocal_index = R1_tmp;
1853   const Register Rval = R0_tmp;
1854 
1855   // get constant in Rconst
1856   __ ldrsb(R2_tmp, at_bcp(4));
1857   __ ldrb(R3_tmp, at_bcp(5));
1858   __ orr(Rconst, R3_tmp, AsmOperand(R2_tmp, lsl, 8));
1859 
1860   locals_index_wide(Rlocal_index);
1861   Address local = load_iaddress(Rlocal_index, Rtemp);
1862   __ ldr_s32(Rval, local);
1863   __ add(Rval, Rval, Rconst);
1864   __ str_32(Rval, local);
1865 }
1866 
1867 
1868 void TemplateTable::convert() {
1869   // Checking
1870 #ifdef ASSERT
1871   { TosState tos_in  = ilgl;
1872     TosState tos_out = ilgl;
1873     switch (bytecode()) {
1874       case Bytecodes::_i2l: // fall through
1875       case Bytecodes::_i2f: // fall through
1876       case Bytecodes::_i2d: // fall through
1877       case Bytecodes::_i2b: // fall through
1878       case Bytecodes::_i2c: // fall through
1879       case Bytecodes::_i2s: tos_in = itos; break;
1880       case Bytecodes::_l2i: // fall through
1881       case Bytecodes::_l2f: // fall through
1882       case Bytecodes::_l2d: tos_in = ltos; break;
1883       case Bytecodes::_f2i: // fall through
1884       case Bytecodes::_f2l: // fall through
1885       case Bytecodes::_f2d: tos_in = ftos; break;
1886       case Bytecodes::_d2i: // fall through
1887       case Bytecodes::_d2l: // fall through
1888       case Bytecodes::_d2f: tos_in = dtos; break;
1889       default             : ShouldNotReachHere();
1890     }
1891     switch (bytecode()) {
1892       case Bytecodes::_l2i: // fall through
1893       case Bytecodes::_f2i: // fall through
1894       case Bytecodes::_d2i: // fall through
1895       case Bytecodes::_i2b: // fall through
1896       case Bytecodes::_i2c: // fall through
1897       case Bytecodes::_i2s: tos_out = itos; break;
1898       case Bytecodes::_i2l: // fall through
1899       case Bytecodes::_f2l: // fall through
1900       case Bytecodes::_d2l: tos_out = ltos; break;
1901       case Bytecodes::_i2f: // fall through
1902       case Bytecodes::_l2f: // fall through
1903       case Bytecodes::_d2f: tos_out = ftos; break;
1904       case Bytecodes::_i2d: // fall through
1905       case Bytecodes::_l2d: // fall through
1906       case Bytecodes::_f2d: tos_out = dtos; break;
1907       default             : ShouldNotReachHere();
1908     }
1909     transition(tos_in, tos_out);
1910   }
1911 #endif // ASSERT
1912 
1913   // Conversion
1914   switch (bytecode()) {
1915     case Bytecodes::_i2l:
1916 #ifdef AARCH64
1917       __ sign_extend(R0_tos, R0_tos, 32);
1918 #else
1919       __ mov(R1_tos_hi, AsmOperand(R0_tos, asr, BitsPerWord-1));
1920 #endif // AARCH64
1921       break;
1922 
1923     case Bytecodes::_i2f:
1924 #ifdef AARCH64
1925       __ scvtf_sw(S0_tos, R0_tos);
1926 #else
1927 #ifdef __SOFTFP__
1928       __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_i2f), R0_tos);
1929 #else
1930       __ fmsr(S0_tmp, R0_tos);
1931       __ fsitos(S0_tos, S0_tmp);
1932 #endif // __SOFTFP__
1933 #endif // AARCH64
1934       break;
1935 
1936     case Bytecodes::_i2d:
1937 #ifdef AARCH64
1938       __ scvtf_dw(D0_tos, R0_tos);
1939 #else
1940 #ifdef __SOFTFP__
1941       __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_i2d), R0_tos);
1942 #else
1943       __ fmsr(S0_tmp, R0_tos);
1944       __ fsitod(D0_tos, S0_tmp);
1945 #endif // __SOFTFP__
1946 #endif // AARCH64
1947       break;
1948 
1949     case Bytecodes::_i2b:
1950       __ sign_extend(R0_tos, R0_tos, 8);
1951       break;
1952 
1953     case Bytecodes::_i2c:
1954       __ zero_extend(R0_tos, R0_tos, 16);
1955       break;
1956 
1957     case Bytecodes::_i2s:
1958       __ sign_extend(R0_tos, R0_tos, 16);
1959       break;
1960 
1961     case Bytecodes::_l2i:
1962       /* nothing to do */
1963       break;
1964 
1965     case Bytecodes::_l2f:
1966 #ifdef AARCH64
1967       __ scvtf_sx(S0_tos, R0_tos);
1968 #else
1969       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::l2f), R0_tos_lo, R1_tos_hi);
1970 #if !defined(__SOFTFP__) && !defined(__ABI_HARD__)
1971       __ fmsr(S0_tos, R0);
1972 #endif // !__SOFTFP__ && !__ABI_HARD__
1973 #endif // AARCH64
1974       break;
1975 
1976     case Bytecodes::_l2d:
1977 #ifdef AARCH64
1978       __ scvtf_dx(D0_tos, R0_tos);
1979 #else
1980       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::l2d), R0_tos_lo, R1_tos_hi);
1981 #if !defined(__SOFTFP__) && !defined(__ABI_HARD__)
1982       __ fmdrr(D0_tos, R0, R1);
1983 #endif // !__SOFTFP__ && !__ABI_HARD__
1984 #endif // AARCH64
1985       break;
1986 
1987     case Bytecodes::_f2i:
1988 #ifdef AARCH64
1989       __ fcvtzs_ws(R0_tos, S0_tos);
1990 #else
1991 #ifndef __SOFTFP__
1992       __ ftosizs(S0_tos, S0_tos);
1993       __ fmrs(R0_tos, S0_tos);
1994 #else
1995       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i), R0_tos);
1996 #endif // !__SOFTFP__
1997 #endif // AARCH64
1998       break;
1999 
2000     case Bytecodes::_f2l:
2001 #ifdef AARCH64
2002       __ fcvtzs_xs(R0_tos, S0_tos);
2003 #else
2004 #ifndef __SOFTFP__
2005       __ fmrs(R0_tos, S0_tos);
2006 #endif // !__SOFTFP__
2007       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l), R0_tos);
2008 #endif // AARCH64
2009       break;
2010 
2011     case Bytecodes::_f2d:
2012 #ifdef __SOFTFP__
2013       __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_f2d), R0_tos);
2014 #else
2015       __ convert_f2d(D0_tos, S0_tos);
2016 #endif // __SOFTFP__
2017       break;
2018 
2019     case Bytecodes::_d2i:
2020 #ifdef AARCH64
2021       __ fcvtzs_wd(R0_tos, D0_tos);
2022 #else
2023 #ifndef __SOFTFP__
2024       __ ftosizd(Stemp, D0);
2025       __ fmrs(R0, Stemp);
2026 #else
2027       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i), R0_tos_lo, R1_tos_hi);
2028 #endif // !__SOFTFP__
2029 #endif // AARCH64
2030       break;
2031 
2032     case Bytecodes::_d2l:
2033 #ifdef AARCH64
2034       __ fcvtzs_xd(R0_tos, D0_tos);
2035 #else
2036 #ifndef __SOFTFP__
2037       __ fmrrd(R0_tos_lo, R1_tos_hi, D0_tos);
2038 #endif // !__SOFTFP__
2039       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l), R0_tos_lo, R1_tos_hi);
2040 #endif // AARCH64
2041       break;
2042 
2043     case Bytecodes::_d2f:
2044 #ifdef __SOFTFP__
2045       __ call_VM_leaf(CAST_FROM_FN_PTR(address, __aeabi_d2f), R0_tos_lo, R1_tos_hi);
2046 #else
2047       __ convert_d2f(S0_tos, D0_tos);
2048 #endif // __SOFTFP__
2049       break;
2050 
2051     default:
2052       ShouldNotReachHere();
2053   }
2054 }
2055 
2056 
2057 void TemplateTable::lcmp() {
2058   transition(ltos, itos);
2059 #ifdef AARCH64
2060   const Register arg1 = R1_tmp;
2061   const Register arg2 = R0_tos;
2062 
2063   __ pop_l(arg1);
2064 
2065   __ cmp(arg1, arg2);
2066   __ cset(R0_tos, gt);               // 1 if '>', else 0
2067   __ csinv(R0_tos, R0_tos, ZR, ge);  // previous value if '>=', else -1
2068 #else
2069   const Register arg1_lo = R2_tmp;
2070   const Register arg1_hi = R3_tmp;
2071   const Register arg2_lo = R0_tos_lo;
2072   const Register arg2_hi = R1_tos_hi;
2073   const Register res = R4_tmp;
2074 
2075   __ pop_l(arg1_lo, arg1_hi);
2076 
2077   // long compare arg1 with arg2
2078   // result is -1/0/+1 if '<'/'='/'>'
2079   Label done;
2080 
2081   __ mov (res, 0);
2082   __ cmp (arg1_hi, arg2_hi);
2083   __ mvn (res, 0, lt);
2084   __ mov (res, 1, gt);
2085   __ b(done, ne);
2086   __ cmp (arg1_lo, arg2_lo);
2087   __ mvn (res, 0, lo);
2088   __ mov (res, 1, hi);
2089   __ bind(done);
2090   __ mov (R0_tos, res);
2091 #endif // AARCH64
2092 }
2093 
2094 
2095 void TemplateTable::float_cmp(bool is_float, int unordered_result) {
2096   assert((unordered_result == 1) || (unordered_result == -1), "invalid unordered result");
2097 
2098 #ifdef AARCH64
2099   if (is_float) {
2100     transition(ftos, itos);
2101     __ pop_f(S1_tmp);
2102     __ fcmp_s(S1_tmp, S0_tos);
2103   } else {
2104     transition(dtos, itos);
2105     __ pop_d(D1_tmp);
2106     __ fcmp_d(D1_tmp, D0_tos);
2107   }
2108 
2109   if (unordered_result < 0) {
2110     __ cset(R0_tos, gt);               // 1 if '>', else 0
2111     __ csinv(R0_tos, R0_tos, ZR, ge);  // previous value if '>=', else -1
2112   } else {
2113     __ cset(R0_tos, hi);               // 1 if '>' or unordered, else 0
2114     __ csinv(R0_tos, R0_tos, ZR, pl);  // previous value if '>=' or unordered, else -1
2115   }
2116 
2117 #else
2118 
2119 #ifdef __SOFTFP__
2120 
2121   if (is_float) {
2122     transition(ftos, itos);
2123     const Register Rx = R0;
2124     const Register Ry = R1;
2125 
2126     __ mov(Ry, R0_tos);
2127     __ pop_i(Rx);
2128 
2129     if (unordered_result == 1) {
2130       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::fcmpg), Rx, Ry);
2131     } else {
2132       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::fcmpl), Rx, Ry);
2133     }
2134 
2135   } else {
2136 
2137     transition(dtos, itos);
2138     const Register Rx_lo = R0;
2139     const Register Rx_hi = R1;
2140     const Register Ry_lo = R2;
2141     const Register Ry_hi = R3;
2142 
2143     __ mov(Ry_lo, R0_tos_lo);
2144     __ mov(Ry_hi, R1_tos_hi);
2145     __ pop_l(Rx_lo, Rx_hi);
2146 
2147     if (unordered_result == 1) {
2148       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dcmpg), Rx_lo, Rx_hi, Ry_lo, Ry_hi);
2149     } else {
2150       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dcmpl), Rx_lo, Rx_hi, Ry_lo, Ry_hi);
2151     }
2152   }
2153 
2154 #else
2155 
2156   if (is_float) {
2157     transition(ftos, itos);
2158     __ pop_f(S1_tmp);
2159     __ fcmps(S1_tmp, S0_tos);
2160   } else {
2161     transition(dtos, itos);
2162     __ pop_d(D1_tmp);
2163     __ fcmpd(D1_tmp, D0_tos);
2164   }
2165 
2166   __ fmstat();
2167 
2168   // comparison result | flag N | flag Z | flag C | flag V
2169   // "<"               |   1    |   0    |   0    |   0
2170   // "=="              |   0    |   1    |   1    |   0
2171   // ">"               |   0    |   0    |   1    |   0
2172   // unordered         |   0    |   0    |   1    |   1
2173 
2174   if (unordered_result < 0) {
2175     __ mov(R0_tos, 1);           // result ==  1 if greater
2176     __ mvn(R0_tos, 0, lt);       // result == -1 if less or unordered (N!=V)
2177   } else {
2178     __ mov(R0_tos, 1);           // result ==  1 if greater or unordered
2179     __ mvn(R0_tos, 0, mi);       // result == -1 if less (N=1)
2180   }
2181   __ mov(R0_tos, 0, eq);         // result ==  0 if equ (Z=1)
2182 #endif // __SOFTFP__
2183 #endif // AARCH64
2184 }
2185 
2186 
2187 void TemplateTable::branch(bool is_jsr, bool is_wide) {
2188 
2189   const Register Rdisp = R0_tmp;
2190   const Register Rbumped_taken_count = R5_tmp;
2191 
2192   __ profile_taken_branch(R0_tmp, Rbumped_taken_count); // R0 holds updated MDP, Rbumped_taken_count holds bumped taken count
2193 
2194   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
2195                              InvocationCounter::counter_offset();
2196   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
2197                               InvocationCounter::counter_offset();
2198   const int method_offset = frame::interpreter_frame_method_offset * wordSize;
2199 
2200   // Load up R0 with the branch displacement
2201   if (is_wide) {
2202     __ ldrsb(R0_tmp, at_bcp(1));
2203     __ ldrb(R1_tmp, at_bcp(2));
2204     __ ldrb(R2_tmp, at_bcp(3));
2205     __ ldrb(R3_tmp, at_bcp(4));
2206     __ orr(R0_tmp, R1_tmp, AsmOperand(R0_tmp, lsl, BitsPerByte));
2207     __ orr(R0_tmp, R2_tmp, AsmOperand(R0_tmp, lsl, BitsPerByte));
2208     __ orr(Rdisp, R3_tmp, AsmOperand(R0_tmp, lsl, BitsPerByte));
2209   } else {
2210     __ ldrsb(R0_tmp, at_bcp(1));
2211     __ ldrb(R1_tmp, at_bcp(2));
2212     __ orr(Rdisp, R1_tmp, AsmOperand(R0_tmp, lsl, BitsPerByte));
2213   }
2214 
2215   // Handle all the JSR stuff here, then exit.
2216   // It's much shorter and cleaner than intermingling with the
2217   // non-JSR normal-branch stuff occuring below.
2218   if (is_jsr) {
2219     // compute return address as bci in R1
2220     const Register Rret_addr = R1_tmp;
2221     assert_different_registers(Rdisp, Rret_addr, Rtemp);
2222 
2223     __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
2224     __ sub(Rret_addr, Rbcp, - (is_wide ? 5 : 3) + in_bytes(ConstMethod::codes_offset()));
2225     __ sub(Rret_addr, Rret_addr, Rtemp);
2226 
2227     // Load the next target bytecode into R3_bytecode and advance Rbcp
2228 #ifdef AARCH64
2229     __ add(Rbcp, Rbcp, Rdisp);
2230     __ ldrb(R3_bytecode, Address(Rbcp));
2231 #else
2232     __ ldrb(R3_bytecode, Address(Rbcp, Rdisp, lsl, 0, pre_indexed));
2233 #endif // AARCH64
2234 
2235     // Push return address
2236     __ push_i(Rret_addr);
2237     // jsr returns vtos
2238     __ dispatch_only_noverify(vtos);
2239     return;
2240   }
2241 
2242   // Normal (non-jsr) branch handling
2243 
2244   // Adjust the bcp by the displacement in Rdisp and load next bytecode.
2245 #ifdef AARCH64
2246   __ add(Rbcp, Rbcp, Rdisp);
2247   __ ldrb(R3_bytecode, Address(Rbcp));
2248 #else
2249   __ ldrb(R3_bytecode, Address(Rbcp, Rdisp, lsl, 0, pre_indexed));
2250 #endif // AARCH64
2251 
2252   assert(UseLoopCounter || !UseOnStackReplacement, "on-stack-replacement requires loop counters");
2253   Label backedge_counter_overflow;
2254   Label profile_method;
2255   Label dispatch;
2256 
2257   if (UseLoopCounter) {
2258     // increment backedge counter for backward branches
2259     // Rdisp (R0): target offset
2260 
2261     const Register Rcnt = R2_tmp;
2262     const Register Rcounters = R1_tmp;
2263 
2264     // count only if backward branch
2265 #ifdef AARCH64
2266     __ tbz(Rdisp, (BitsPerWord - 1), dispatch); // TODO-AARCH64: check performance of this variant on 32-bit ARM
2267 #else
2268     __ tst(Rdisp, Rdisp);
2269     __ b(dispatch, pl);
2270 #endif // AARCH64
2271 
2272     if (TieredCompilation) {
2273       Label no_mdo;
2274       int increment = InvocationCounter::count_increment;
2275       if (ProfileInterpreter) {
2276         // Are we profiling?
2277         __ ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
2278         __ cbz(Rtemp, no_mdo);
2279         // Increment the MDO backedge counter
2280         const Address mdo_backedge_counter(Rtemp, in_bytes(MethodData::backedge_counter_offset()) +
2281                                                   in_bytes(InvocationCounter::counter_offset()));
2282         const Address mask(Rtemp, in_bytes(MethodData::backedge_mask_offset()));
2283         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
2284                                    Rcnt, R4_tmp, eq, &backedge_counter_overflow);
2285         __ b(dispatch);
2286       }
2287       __ bind(no_mdo);
2288       // Increment backedge counter in MethodCounters*
2289       // Note Rbumped_taken_count is a callee saved registers for ARM32, but caller saved for ARM64
2290       __ get_method_counters(Rmethod, Rcounters, dispatch, true /*saveRegs*/,
2291                              Rdisp, R3_bytecode,
2292                              AARCH64_ONLY(Rbumped_taken_count) NOT_AARCH64(noreg));
2293       const Address mask(Rcounters, in_bytes(MethodCounters::backedge_mask_offset()));
2294       __ increment_mask_and_jump(Address(Rcounters, be_offset), increment, mask,
2295                                  Rcnt, R4_tmp, eq, &backedge_counter_overflow);
2296     } else {
2297       // Increment backedge counter in MethodCounters*
2298       __ get_method_counters(Rmethod, Rcounters, dispatch, true /*saveRegs*/,
2299                              Rdisp, R3_bytecode,
2300                              AARCH64_ONLY(Rbumped_taken_count) NOT_AARCH64(noreg));
2301       __ ldr_u32(Rtemp, Address(Rcounters, be_offset));           // load backedge counter
2302       __ add(Rtemp, Rtemp, InvocationCounter::count_increment);   // increment counter
2303       __ str_32(Rtemp, Address(Rcounters, be_offset));            // store counter
2304 
2305       __ ldr_u32(Rcnt, Address(Rcounters, inv_offset));           // load invocation counter
2306 #ifdef AARCH64
2307       __ andr(Rcnt, Rcnt, (unsigned int)InvocationCounter::count_mask_value);  // and the status bits
2308 #else
2309       __ bic(Rcnt, Rcnt, ~InvocationCounter::count_mask_value);  // and the status bits
2310 #endif // AARCH64
2311       __ add(Rcnt, Rcnt, Rtemp);                                 // add both counters
2312 
2313       if (ProfileInterpreter) {
2314         // Test to see if we should create a method data oop
2315         const Address profile_limit(Rcounters, in_bytes(MethodCounters::interpreter_profile_limit_offset()));
2316         __ ldr_s32(Rtemp, profile_limit);
2317         __ cmp_32(Rcnt, Rtemp);
2318         __ b(dispatch, lt);
2319 
2320         // if no method data exists, go to profile method
2321         __ test_method_data_pointer(R4_tmp, profile_method);
2322 
2323         if (UseOnStackReplacement) {
2324           // check for overflow against Rbumped_taken_count, which is the MDO taken count
2325           const Address backward_branch_limit(Rcounters, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset()));
2326           __ ldr_s32(Rtemp, backward_branch_limit);
2327           __ cmp(Rbumped_taken_count, Rtemp);
2328           __ b(dispatch, lo);
2329 
2330           // When ProfileInterpreter is on, the backedge_count comes from the
2331           // MethodData*, which value does not get reset on the call to
2332           // frequency_counter_overflow().  To avoid excessive calls to the overflow
2333           // routine while the method is being compiled, add a second test to make
2334           // sure the overflow function is called only once every overflow_frequency.
2335           const int overflow_frequency = 1024;
2336 
2337 #ifdef AARCH64
2338           __ tst(Rbumped_taken_count, (unsigned)(overflow_frequency-1));
2339 #else
2340           // was '__ andrs(...,overflow_frequency-1)', testing if lowest 10 bits are 0
2341           assert(overflow_frequency == (1 << 10),"shift by 22 not correct for expected frequency");
2342           __ movs(Rbumped_taken_count, AsmOperand(Rbumped_taken_count, lsl, 22));
2343 #endif // AARCH64
2344 
2345           __ b(backedge_counter_overflow, eq);
2346         }
2347       } else {
2348         if (UseOnStackReplacement) {
2349           // check for overflow against Rcnt, which is the sum of the counters
2350           const Address backward_branch_limit(Rcounters, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset()));
2351           __ ldr_s32(Rtemp, backward_branch_limit);
2352           __ cmp_32(Rcnt, Rtemp);
2353           __ b(backedge_counter_overflow, hs);
2354 
2355         }
2356       }
2357     }
2358     __ bind(dispatch);
2359   }
2360 
2361   if (!UseOnStackReplacement) {
2362     __ bind(backedge_counter_overflow);
2363   }
2364 
2365   // continue with the bytecode @ target
2366   __ dispatch_only(vtos);
2367 
2368   if (UseLoopCounter) {
2369     if (ProfileInterpreter) {
2370       // Out-of-line code to allocate method data oop.
2371       __ bind(profile_method);
2372 
2373       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
2374       __ set_method_data_pointer_for_bcp();
2375       // reload next bytecode
2376       __ ldrb(R3_bytecode, Address(Rbcp));
2377       __ b(dispatch);
2378     }
2379 
2380     if (UseOnStackReplacement) {
2381       // invocation counter overflow
2382       __ bind(backedge_counter_overflow);
2383 
2384       __ sub(R1, Rbcp, Rdisp);                   // branch bcp
2385       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), R1);
2386 
2387       // R0: osr nmethod (osr ok) or NULL (osr not possible)
2388       const Register Rnmethod = R0;
2389 
2390       __ ldrb(R3_bytecode, Address(Rbcp));       // reload next bytecode
2391 
2392       __ cbz(Rnmethod, dispatch);                // test result, no osr if null
2393 
2394       // nmethod may have been invalidated (VM may block upon call_VM return)
2395       __ ldrb(R1_tmp, Address(Rnmethod, nmethod::state_offset()));
2396       __ cmp(R1_tmp, nmethod::in_use);
2397       __ b(dispatch, ne);
2398 
2399       // We have the address of an on stack replacement routine in Rnmethod,
2400       // We need to prepare to execute the OSR method. First we must
2401       // migrate the locals and monitors off of the stack.
2402 
2403       __ mov(Rtmp_save0, Rnmethod);                      // save the nmethod
2404 
2405       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2406 
2407       // R0 is OSR buffer
2408 
2409       __ ldr(R1_tmp, Address(Rtmp_save0, nmethod::osr_entry_point_offset()));
2410       __ ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
2411 
2412 #ifdef AARCH64
2413       __ ldp(FP, LR, Address(FP));
2414       __ mov(SP, Rtemp);
2415 #else
2416       __ ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
2417       __ bic(SP, Rtemp, StackAlignmentInBytes - 1);     // Remove frame and align stack
2418 #endif // AARCH64
2419 
2420       __ jump(R1_tmp);
2421     }
2422   }
2423 }
2424 
2425 
2426 void TemplateTable::if_0cmp(Condition cc) {
2427   transition(itos, vtos);
2428   // assume branch is more often taken than not (loops use backward branches)
2429   Label not_taken;
2430 #ifdef AARCH64
2431   if (cc == equal) {
2432     __ cbnz_w(R0_tos, not_taken);
2433   } else if (cc == not_equal) {
2434     __ cbz_w(R0_tos, not_taken);
2435   } else {
2436     __ cmp_32(R0_tos, 0);
2437     __ b(not_taken, convNegCond(cc));
2438   }
2439 #else
2440   __ cmp_32(R0_tos, 0);
2441   __ b(not_taken, convNegCond(cc));
2442 #endif // AARCH64
2443   branch(false, false);
2444   __ bind(not_taken);
2445   __ profile_not_taken_branch(R0_tmp);
2446 }
2447 
2448 
2449 void TemplateTable::if_icmp(Condition cc) {
2450   transition(itos, vtos);
2451   // assume branch is more often taken than not (loops use backward branches)
2452   Label not_taken;
2453   __ pop_i(R1_tmp);
2454   __ cmp_32(R1_tmp, R0_tos);
2455   __ b(not_taken, convNegCond(cc));
2456   branch(false, false);
2457   __ bind(not_taken);
2458   __ profile_not_taken_branch(R0_tmp);
2459 }
2460 
2461 
2462 void TemplateTable::if_nullcmp(Condition cc) {
2463   transition(atos, vtos);
2464   assert(cc == equal || cc == not_equal, "invalid condition");
2465 
2466   // assume branch is more often taken than not (loops use backward branches)
2467   Label not_taken;
2468   if (cc == equal) {
2469     __ cbnz(R0_tos, not_taken);
2470   } else {
2471     __ cbz(R0_tos, not_taken);
2472   }
2473   branch(false, false);
2474   __ bind(not_taken);
2475   __ profile_not_taken_branch(R0_tmp);
2476 }
2477 
2478 
2479 void TemplateTable::if_acmp(Condition cc) {
2480   transition(atos, vtos);
2481   // assume branch is more often taken than not (loops use backward branches)
2482   Label not_taken;
2483   __ pop_ptr(R1_tmp);
2484   __ cmp(R1_tmp, R0_tos);
2485   __ b(not_taken, convNegCond(cc));
2486   branch(false, false);
2487   __ bind(not_taken);
2488   __ profile_not_taken_branch(R0_tmp);
2489 }
2490 
2491 
2492 void TemplateTable::ret() {
2493   transition(vtos, vtos);
2494   const Register Rlocal_index = R1_tmp;
2495   const Register Rret_bci = Rtmp_save0; // R4/R19
2496 
2497   locals_index(Rlocal_index);
2498   Address local = load_iaddress(Rlocal_index, Rtemp);
2499   __ ldr_s32(Rret_bci, local);          // get return bci, compute return bcp
2500   __ profile_ret(Rtmp_save1, Rret_bci);
2501   __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
2502   __ add(Rtemp, Rtemp, in_bytes(ConstMethod::codes_offset()));
2503   __ add(Rbcp, Rtemp, Rret_bci);
2504   __ dispatch_next(vtos);
2505 }
2506 
2507 
2508 void TemplateTable::wide_ret() {
2509   transition(vtos, vtos);
2510   const Register Rlocal_index = R1_tmp;
2511   const Register Rret_bci = Rtmp_save0; // R4/R19
2512 
2513   locals_index_wide(Rlocal_index);
2514   Address local = load_iaddress(Rlocal_index, Rtemp);
2515   __ ldr_s32(Rret_bci, local);               // get return bci, compute return bcp
2516   __ profile_ret(Rtmp_save1, Rret_bci);
2517   __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
2518   __ add(Rtemp, Rtemp, in_bytes(ConstMethod::codes_offset()));
2519   __ add(Rbcp, Rtemp, Rret_bci);
2520   __ dispatch_next(vtos);
2521 }
2522 
2523 
2524 void TemplateTable::tableswitch() {
2525   transition(itos, vtos);
2526 
2527   const Register Rindex  = R0_tos;
2528 #ifndef AARCH64
2529   const Register Rtemp2  = R1_tmp;
2530 #endif // !AARCH64
2531   const Register Rabcp   = R2_tmp;  // aligned bcp
2532   const Register Rlow    = R3_tmp;
2533   const Register Rhigh   = R4_tmp;
2534   const Register Roffset = R5_tmp;
2535 
2536   // align bcp
2537   __ add(Rtemp, Rbcp, 1 + (2*BytesPerInt-1));
2538   __ align_reg(Rabcp, Rtemp, BytesPerInt);
2539 
2540   // load lo & hi
2541 #ifdef AARCH64
2542   __ ldp_w(Rlow, Rhigh, Address(Rabcp, 2*BytesPerInt, post_indexed));
2543 #else
2544   __ ldmia(Rabcp, RegisterSet(Rlow) | RegisterSet(Rhigh), writeback);
2545 #endif // AARCH64
2546   __ byteswap_u32(Rlow, Rtemp, Rtemp2);
2547   __ byteswap_u32(Rhigh, Rtemp, Rtemp2);
2548 
2549   // compare index with high bound
2550   __ cmp_32(Rhigh, Rindex);
2551 
2552 #ifdef AARCH64
2553   Label default_case, do_dispatch;
2554   __ ccmp_w(Rindex, Rlow, Assembler::flags_for_condition(lt), ge);
2555   __ b(default_case, lt);
2556 
2557   __ sub_w(Rindex, Rindex, Rlow);
2558   __ ldr_s32(Roffset, Address(Rabcp, Rindex, ex_sxtw, LogBytesPerInt));
2559   if(ProfileInterpreter) {
2560     __ sxtw(Rindex, Rindex);
2561     __ profile_switch_case(Rabcp, Rindex, Rtemp2, R0_tmp);
2562   }
2563   __ b(do_dispatch);
2564 
2565   __ bind(default_case);
2566   __ ldr_s32(Roffset, Address(Rabcp, -3 * BytesPerInt));
2567   if(ProfileInterpreter) {
2568     __ profile_switch_default(R0_tmp);
2569   }
2570 
2571   __ bind(do_dispatch);
2572 #else
2573 
2574   // if Rindex <= Rhigh then calculate index in table (Rindex - Rlow)
2575   __ subs(Rindex, Rindex, Rlow, ge);
2576 
2577   // if Rindex <= Rhigh and (Rindex - Rlow) >= 0
2578   // ("ge" status accumulated from cmp and subs instructions) then load
2579   // offset from table, otherwise load offset for default case
2580 
2581   if(ProfileInterpreter) {
2582     Label default_case, continue_execution;
2583 
2584     __ b(default_case, lt);
2585     __ ldr(Roffset, Address(Rabcp, Rindex, lsl, LogBytesPerInt));
2586     __ profile_switch_case(Rabcp, Rindex, Rtemp2, R0_tmp);
2587     __ b(continue_execution);
2588 
2589     __ bind(default_case);
2590     __ profile_switch_default(R0_tmp);
2591     __ ldr(Roffset, Address(Rabcp, -3 * BytesPerInt));
2592 
2593     __ bind(continue_execution);
2594   } else {
2595     __ ldr(Roffset, Address(Rabcp, -3 * BytesPerInt), lt);
2596     __ ldr(Roffset, Address(Rabcp, Rindex, lsl, LogBytesPerInt), ge);
2597   }
2598 #endif // AARCH64
2599 
2600   __ byteswap_u32(Roffset, Rtemp, Rtemp2);
2601 
2602   // load the next bytecode to R3_bytecode and advance Rbcp
2603 #ifdef AARCH64
2604   __ add(Rbcp, Rbcp, Roffset, ex_sxtw);
2605   __ ldrb(R3_bytecode, Address(Rbcp));
2606 #else
2607   __ ldrb(R3_bytecode, Address(Rbcp, Roffset, lsl, 0, pre_indexed));
2608 #endif // AARCH64
2609   __ dispatch_only(vtos);
2610 
2611 }
2612 
2613 
2614 void TemplateTable::lookupswitch() {
2615   transition(itos, itos);
2616   __ stop("lookupswitch bytecode should have been rewritten");
2617 }
2618 
2619 
2620 void TemplateTable::fast_linearswitch() {
2621   transition(itos, vtos);
2622   Label loop, found, default_case, continue_execution;
2623 
2624   const Register Rkey     = R0_tos;
2625   const Register Rabcp    = R2_tmp;  // aligned bcp
2626   const Register Rdefault = R3_tmp;
2627   const Register Rcount   = R4_tmp;
2628   const Register Roffset  = R5_tmp;
2629 
2630   // bswap Rkey, so we can avoid bswapping the table entries
2631   __ byteswap_u32(Rkey, R1_tmp, Rtemp);
2632 
2633   // align bcp
2634   __ add(Rtemp, Rbcp, 1 + (BytesPerInt-1));
2635   __ align_reg(Rabcp, Rtemp, BytesPerInt);
2636 
2637   // load default & counter
2638 #ifdef AARCH64
2639   __ ldp_w(Rdefault, Rcount, Address(Rabcp, 2*BytesPerInt, post_indexed));
2640 #else
2641   __ ldmia(Rabcp, RegisterSet(Rdefault) | RegisterSet(Rcount), writeback);
2642 #endif // AARCH64
2643   __ byteswap_u32(Rcount, R1_tmp, Rtemp);
2644 
2645 #ifdef AARCH64
2646   __ cbz_w(Rcount, default_case);
2647 #else
2648   __ cmp_32(Rcount, 0);
2649   __ ldr(Rtemp, Address(Rabcp, 2*BytesPerInt, post_indexed), ne);
2650   __ b(default_case, eq);
2651 #endif // AARCH64
2652 
2653   // table search
2654   __ bind(loop);
2655 #ifdef AARCH64
2656   __ ldr_s32(Rtemp, Address(Rabcp, 2*BytesPerInt, post_indexed));
2657 #endif // AARCH64
2658   __ cmp_32(Rtemp, Rkey);
2659   __ b(found, eq);
2660   __ subs(Rcount, Rcount, 1);
2661 #ifndef AARCH64
2662   __ ldr(Rtemp, Address(Rabcp, 2*BytesPerInt, post_indexed), ne);
2663 #endif // !AARCH64
2664   __ b(loop, ne);
2665 
2666   // default case
2667   __ bind(default_case);
2668   __ profile_switch_default(R0_tmp);
2669   __ mov(Roffset, Rdefault);
2670   __ b(continue_execution);
2671 
2672   // entry found -> get offset
2673   __ bind(found);
2674   // Rabcp is already incremented and points to the next entry
2675   __ ldr_s32(Roffset, Address(Rabcp, -BytesPerInt));
2676   if (ProfileInterpreter) {
2677     // Calculate index of the selected case.
2678     assert_different_registers(Roffset, Rcount, Rtemp, R0_tmp, R1_tmp, R2_tmp);
2679 
2680     // align bcp
2681     __ add(Rtemp, Rbcp, 1 + (BytesPerInt-1));
2682     __ align_reg(R2_tmp, Rtemp, BytesPerInt);
2683 
2684     // load number of cases
2685     __ ldr_u32(R2_tmp, Address(R2_tmp, BytesPerInt));
2686     __ byteswap_u32(R2_tmp, R1_tmp, Rtemp);
2687 
2688     // Selected index = <number of cases> - <current loop count>
2689     __ sub(R1_tmp, R2_tmp, Rcount);
2690     __ profile_switch_case(R0_tmp, R1_tmp, Rtemp, R1_tmp);
2691   }
2692 
2693   // continue execution
2694   __ bind(continue_execution);
2695   __ byteswap_u32(Roffset, R1_tmp, Rtemp);
2696 
2697   // load the next bytecode to R3_bytecode and advance Rbcp
2698 #ifdef AARCH64
2699   __ add(Rbcp, Rbcp, Roffset, ex_sxtw);
2700   __ ldrb(R3_bytecode, Address(Rbcp));
2701 #else
2702   __ ldrb(R3_bytecode, Address(Rbcp, Roffset, lsl, 0, pre_indexed));
2703 #endif // AARCH64
2704   __ dispatch_only(vtos);
2705 }
2706 
2707 
2708 void TemplateTable::fast_binaryswitch() {
2709   transition(itos, vtos);
2710   // Implementation using the following core algorithm:
2711   //
2712   // int binary_search(int key, LookupswitchPair* array, int n) {
2713   //   // Binary search according to "Methodik des Programmierens" by
2714   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2715   //   int i = 0;
2716   //   int j = n;
2717   //   while (i+1 < j) {
2718   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2719   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2720   //     // where a stands for the array and assuming that the (inexisting)
2721   //     // element a[n] is infinitely big.
2722   //     int h = (i + j) >> 1;
2723   //     // i < h < j
2724   //     if (key < array[h].fast_match()) {
2725   //       j = h;
2726   //     } else {
2727   //       i = h;
2728   //     }
2729   //   }
2730   //   // R: a[i] <= key < a[i+1] or Q
2731   //   // (i.e., if key is within array, i is the correct index)
2732   //   return i;
2733   // }
2734 
2735   // register allocation
2736   const Register key    = R0_tos;                // already set (tosca)
2737   const Register array  = R1_tmp;
2738   const Register i      = R2_tmp;
2739   const Register j      = R3_tmp;
2740   const Register h      = R4_tmp;
2741   const Register val    = R5_tmp;
2742   const Register temp1  = Rtemp;
2743   const Register temp2  = LR_tmp;
2744   const Register offset = R3_tmp;
2745 
2746   // set 'array' = aligned bcp + 2 ints
2747   __ add(temp1, Rbcp, 1 + (BytesPerInt-1) + 2*BytesPerInt);
2748   __ align_reg(array, temp1, BytesPerInt);
2749 
2750   // initialize i & j
2751   __ mov(i, 0);                                  // i = 0;
2752   __ ldr_s32(j, Address(array, -BytesPerInt));   // j = length(array);
2753   // Convert j into native byteordering
2754   __ byteswap_u32(j, temp1, temp2);
2755 
2756   // and start
2757   Label entry;
2758   __ b(entry);
2759 
2760   // binary search loop
2761   { Label loop;
2762     __ bind(loop);
2763     // int h = (i + j) >> 1;
2764     __ add(h, i, j);                             // h = i + j;
2765     __ logical_shift_right(h, h, 1);             // h = (i + j) >> 1;
2766     // if (key < array[h].fast_match()) {
2767     //   j = h;
2768     // } else {
2769     //   i = h;
2770     // }
2771 #ifdef AARCH64
2772     __ add(temp1, array, AsmOperand(h, lsl, 1+LogBytesPerInt));
2773     __ ldr_s32(val, Address(temp1));
2774 #else
2775     __ ldr_s32(val, Address(array, h, lsl, 1+LogBytesPerInt));
2776 #endif // AARCH64
2777     // Convert array[h].match to native byte-ordering before compare
2778     __ byteswap_u32(val, temp1, temp2);
2779     __ cmp_32(key, val);
2780     __ mov(j, h, lt);   // j = h if (key <  array[h].fast_match())
2781     __ mov(i, h, ge);   // i = h if (key >= array[h].fast_match())
2782     // while (i+1 < j)
2783     __ bind(entry);
2784     __ add(temp1, i, 1);                             // i+1
2785     __ cmp(temp1, j);                                // i+1 < j
2786     __ b(loop, lt);
2787   }
2788 
2789   // end of binary search, result index is i (must check again!)
2790   Label default_case;
2791   // Convert array[i].match to native byte-ordering before compare
2792 #ifdef AARCH64
2793   __ add(temp1, array, AsmOperand(i, lsl, 1+LogBytesPerInt));
2794   __ ldr_s32(val, Address(temp1));
2795 #else
2796   __ ldr_s32(val, Address(array, i, lsl, 1+LogBytesPerInt));
2797 #endif // AARCH64
2798   __ byteswap_u32(val, temp1, temp2);
2799   __ cmp_32(key, val);
2800   __ b(default_case, ne);
2801 
2802   // entry found
2803   __ add(temp1, array, AsmOperand(i, lsl, 1+LogBytesPerInt));
2804   __ ldr_s32(offset, Address(temp1, 1*BytesPerInt));
2805   __ profile_switch_case(R0, i, R1, i);
2806   __ byteswap_u32(offset, temp1, temp2);
2807 #ifdef AARCH64
2808   __ add(Rbcp, Rbcp, offset, ex_sxtw);
2809   __ ldrb(R3_bytecode, Address(Rbcp));
2810 #else
2811   __ ldrb(R3_bytecode, Address(Rbcp, offset, lsl, 0, pre_indexed));
2812 #endif // AARCH64
2813   __ dispatch_only(vtos);
2814 
2815   // default case
2816   __ bind(default_case);
2817   __ profile_switch_default(R0);
2818   __ ldr_s32(offset, Address(array, -2*BytesPerInt));
2819   __ byteswap_u32(offset, temp1, temp2);
2820 #ifdef AARCH64
2821   __ add(Rbcp, Rbcp, offset, ex_sxtw);
2822   __ ldrb(R3_bytecode, Address(Rbcp));
2823 #else
2824   __ ldrb(R3_bytecode, Address(Rbcp, offset, lsl, 0, pre_indexed));
2825 #endif // AARCH64
2826   __ dispatch_only(vtos);
2827 }
2828 
2829 
2830 void TemplateTable::_return(TosState state) {
2831   transition(state, state);
2832   assert(_desc->calls_vm(), "inconsistent calls_vm information"); // call in remove_activation
2833 
2834   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2835     Label skip_register_finalizer;
2836     assert(state == vtos, "only valid state");
2837     __ ldr(R1, aaddress(0));
2838     __ load_klass(Rtemp, R1);
2839     __ ldr_u32(Rtemp, Address(Rtemp, Klass::access_flags_offset()));
2840     __ tbz(Rtemp, exact_log2(JVM_ACC_HAS_FINALIZER), skip_register_finalizer);
2841 
2842     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), R1);
2843 
2844     __ bind(skip_register_finalizer);
2845   }
2846 
2847   // Explicitly reset last_sp, for handling special case in TemplateInterpreter::deopt_reexecute_entry
2848 #ifndef AARCH64
2849   if (state == vtos) {
2850     __ mov(Rtemp, 0);
2851     __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
2852   }
2853 #endif
2854 
2855   // Narrow result if state is itos but result type is smaller.
2856   // Need to narrow in the return bytecode rather than in generate_return_entry
2857   // since compiled code callers expect the result to already be narrowed.
2858   if (state == itos) {
2859     __ narrow(R0_tos);
2860   }
2861   __ remove_activation(state, LR);
2862 
2863   __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
2864 
2865 #ifndef AARCH64
2866   // According to interpreter calling conventions, result is returned in R0/R1,
2867   // so ftos (S0) and dtos (D0) are moved to R0/R1.
2868   // This conversion should be done after remove_activation, as it uses
2869   // push(state) & pop(state) to preserve return value.
2870   __ convert_tos_to_retval(state);
2871 #endif // !AARCH64
2872 
2873   __ ret();
2874 
2875   __ nop(); // to avoid filling CPU pipeline with invalid instructions
2876   __ nop();
2877 }
2878 
2879 
2880 // ----------------------------------------------------------------------------
2881 // Volatile variables demand their effects be made known to all CPU's in
2882 // order.  Store buffers on most chips allow reads & writes to reorder; the
2883 // JMM's ReadAfterWrite.java test fails in -Xint mode without some kind of
2884 // memory barrier (i.e., it's not sufficient that the interpreter does not
2885 // reorder volatile references, the hardware also must not reorder them).
2886 //
2887 // According to the new Java Memory Model (JMM):
2888 // (1) All volatiles are serialized wrt to each other.
2889 // ALSO reads & writes act as aquire & release, so:
2890 // (2) A read cannot let unrelated NON-volatile memory refs that happen after
2891 // the read float up to before the read.  It's OK for non-volatile memory refs
2892 // that happen before the volatile read to float down below it.
2893 // (3) Similar a volatile write cannot let unrelated NON-volatile memory refs
2894 // that happen BEFORE the write float down to after the write.  It's OK for
2895 // non-volatile memory refs that happen after the volatile write to float up
2896 // before it.
2897 //
2898 // We only put in barriers around volatile refs (they are expensive), not
2899 // _between_ memory refs (that would require us to track the flavor of the
2900 // previous memory refs).  Requirements (2) and (3) require some barriers
2901 // before volatile stores and after volatile loads.  These nearly cover
2902 // requirement (1) but miss the volatile-store-volatile-load case.  This final
2903 // case is placed after volatile-stores although it could just as well go
2904 // before volatile-loads.
2905 // TODO-AARCH64: consider removing extra unused parameters
2906 void TemplateTable::volatile_barrier(MacroAssembler::Membar_mask_bits order_constraint,
2907                                      Register tmp,
2908                                      bool preserve_flags,
2909                                      Register load_tgt) {
2910 #ifdef AARCH64
2911   __ membar(order_constraint);
2912 #else
2913   __ membar(order_constraint, tmp, preserve_flags, load_tgt);
2914 #endif
2915 }
2916 
2917 // Blows all volatile registers: R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR.
2918 void TemplateTable::resolve_cache_and_index(int byte_no,
2919                                             Register Rcache,
2920                                             Register Rindex,
2921                                             size_t index_size) {
2922   assert_different_registers(Rcache, Rindex, Rtemp);
2923 
2924   Label resolved;
2925   Bytecodes::Code code = bytecode();
2926   switch (code) {
2927   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2928   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2929   }
2930 
2931   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2932   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, Rindex, Rtemp, byte_no, 1, index_size);
2933   __ cmp(Rtemp, code);  // have we resolved this bytecode?
2934   __ b(resolved, eq);
2935 
2936   // resolve first time through
2937   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2938   __ mov(R1, code);
2939   __ call_VM(noreg, entry, R1);
2940   // Update registers with resolved info
2941   __ get_cache_and_index_at_bcp(Rcache, Rindex, 1, index_size);
2942   __ bind(resolved);
2943 }
2944 
2945 
2946 // The Rcache and Rindex registers must be set before call
2947 void TemplateTable::load_field_cp_cache_entry(Register Rcache,
2948                                               Register Rindex,
2949                                               Register Roffset,
2950                                               Register Rflags,
2951                                               Register Robj,
2952                                               bool is_static = false) {
2953 
2954   assert_different_registers(Rcache, Rindex, Rtemp);
2955   assert_different_registers(Roffset, Rflags, Robj, Rtemp);
2956 
2957   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2958 
2959   __ add(Rtemp, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
2960 
2961   // Field offset
2962   __ ldr(Roffset, Address(Rtemp,
2963            cp_base_offset + ConstantPoolCacheEntry::f2_offset()));
2964 
2965   // Flags
2966   __ ldr_u32(Rflags, Address(Rtemp,
2967            cp_base_offset + ConstantPoolCacheEntry::flags_offset()));
2968 
2969   if (is_static) {
2970     __ ldr(Robj, Address(Rtemp,
2971              cp_base_offset + ConstantPoolCacheEntry::f1_offset()));
2972     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2973     __ ldr(Robj, Address(Robj, mirror_offset));
2974   }
2975 }
2976 
2977 
2978 // Blows all volatile registers: R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR.
2979 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2980                                                Register method,
2981                                                Register itable_index,
2982                                                Register flags,
2983                                                bool is_invokevirtual,
2984                                                bool is_invokevfinal/*unused*/,
2985                                                bool is_invokedynamic) {
2986   // setup registers
2987   const Register cache = R2_tmp;
2988   const Register index = R3_tmp;
2989   const Register temp_reg = Rtemp;
2990   assert_different_registers(cache, index, temp_reg);
2991   assert_different_registers(method, itable_index, temp_reg);
2992 
2993   // determine constant pool cache field offsets
2994   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2995   const int method_offset = in_bytes(
2996     ConstantPoolCache::base_offset() +
2997       ((byte_no == f2_byte)
2998        ? ConstantPoolCacheEntry::f2_offset()
2999        : ConstantPoolCacheEntry::f1_offset()
3000       )
3001     );
3002   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
3003                                     ConstantPoolCacheEntry::flags_offset());
3004   // access constant pool cache fields
3005   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
3006                                     ConstantPoolCacheEntry::f2_offset());
3007 
3008   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
3009   resolve_cache_and_index(byte_no, cache, index, index_size);
3010     __ add(temp_reg, cache, AsmOperand(index, lsl, LogBytesPerWord));
3011     __ ldr(method, Address(temp_reg, method_offset));
3012 
3013   if (itable_index != noreg) {
3014     __ ldr(itable_index, Address(temp_reg, index_offset));
3015   }
3016   __ ldr_u32(flags, Address(temp_reg, flags_offset));
3017 }
3018 
3019 
3020 // The registers cache and index expected to be set before call, and should not be Rtemp.
3021 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR,
3022 // except cache and index registers which are preserved.
3023 void TemplateTable::jvmti_post_field_access(Register Rcache,
3024                                             Register Rindex,
3025                                             bool is_static,
3026                                             bool has_tos) {
3027   assert_different_registers(Rcache, Rindex, Rtemp);
3028 
3029   if (__ can_post_field_access()) {
3030     // Check to see if a field access watch has been set before we take
3031     // the time to call into the VM.
3032 
3033     Label Lcontinue;
3034 
3035     __ ldr_global_s32(Rtemp, (address)JvmtiExport::get_field_access_count_addr());
3036     __ cbz(Rtemp, Lcontinue);
3037 
3038     // cache entry pointer
3039     __ add(R2, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
3040     __ add(R2, R2, in_bytes(ConstantPoolCache::base_offset()));
3041     if (is_static) {
3042       __ mov(R1, 0);        // NULL object reference
3043     } else {
3044       __ pop(atos);         // Get the object
3045       __ mov(R1, R0_tos);
3046       __ verify_oop(R1);
3047       __ push(atos);        // Restore stack state
3048     }
3049     // R1: object pointer or NULL
3050     // R2: cache entry pointer
3051     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access),
3052                R1, R2);
3053     __ get_cache_and_index_at_bcp(Rcache, Rindex, 1);
3054 
3055     __ bind(Lcontinue);
3056   }
3057 }
3058 
3059 
3060 void TemplateTable::pop_and_check_object(Register r) {
3061   __ pop_ptr(r);
3062   __ null_check(r, Rtemp);  // for field access must check obj.
3063   __ verify_oop(r);
3064 }
3065 
3066 
3067 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
3068   transition(vtos, vtos);
3069 
3070   const Register Roffset  = R2_tmp;
3071   const Register Robj     = R3_tmp;
3072   const Register Rcache   = R4_tmp;
3073   const Register Rflagsav = Rtmp_save0;  // R4/R19
3074   const Register Rindex   = R5_tmp;
3075   const Register Rflags   = R5_tmp;
3076 
3077   const bool gen_volatile_check = os::is_MP();
3078 
3079   resolve_cache_and_index(byte_no, Rcache, Rindex, sizeof(u2));
3080   jvmti_post_field_access(Rcache, Rindex, is_static, false);
3081   load_field_cp_cache_entry(Rcache, Rindex, Roffset, Rflags, Robj, is_static);
3082 
3083   if (gen_volatile_check) {
3084     __ mov(Rflagsav, Rflags);
3085   }
3086 
3087   if (!is_static) pop_and_check_object(Robj);
3088 
3089   Label Done, Lint, Ltable, shouldNotReachHere;
3090   Label Lbtos, Lztos, Lctos, Lstos, Litos, Lltos, Lftos, Ldtos, Latos;
3091 
3092   // compute type
3093   __ logical_shift_right(Rflags, Rflags, ConstantPoolCacheEntry::tos_state_shift);
3094   // Make sure we don't need to mask flags after the above shift
3095   ConstantPoolCacheEntry::verify_tos_state_shift();
3096 
3097   // There are actually two versions of implementation of getfield/getstatic:
3098   //
3099   // 32-bit ARM:
3100   // 1) Table switch using add(PC,...) instruction (fast_version)
3101   // 2) Table switch using ldr(PC,...) instruction
3102   //
3103   // AArch64:
3104   // 1) Table switch using adr/add/br instructions (fast_version)
3105   // 2) Table switch using adr/ldr/br instructions
3106   //
3107   // First version requires fixed size of code block for each case and
3108   // can not be used in RewriteBytecodes and VerifyOops
3109   // modes.
3110 
3111   // Size of fixed size code block for fast_version
3112   const int log_max_block_size = 2;
3113   const int max_block_size = 1 << log_max_block_size;
3114 
3115   // Decide if fast version is enabled
3116   bool fast_version = (is_static || !RewriteBytecodes) && !VerifyOops && !VerifyInterpreterStackTop;
3117 
3118   // On 32-bit ARM atos and itos cases can be merged only for fast version, because
3119   // atos requires additional processing in slow version.
3120   // On AArch64 atos and itos cannot be merged.
3121   bool atos_merged_with_itos = AARCH64_ONLY(false) NOT_AARCH64(fast_version);
3122 
3123   assert(number_of_states == 10, "number of tos states should be equal to 9");
3124 
3125   __ cmp(Rflags, itos);
3126 #ifdef AARCH64
3127   __ b(Lint, eq);
3128 
3129   if(fast_version) {
3130     __ adr(Rtemp, Lbtos);
3131     __ add(Rtemp, Rtemp, AsmOperand(Rflags, lsl, log_max_block_size + Assembler::LogInstructionSize));
3132     __ br(Rtemp);
3133   } else {
3134     __ adr(Rtemp, Ltable);
3135     __ ldr(Rtemp, Address::indexed_ptr(Rtemp, Rflags));
3136     __ br(Rtemp);
3137   }
3138 #else
3139   if(atos_merged_with_itos) {
3140     __ cmp(Rflags, atos, ne);
3141   }
3142 
3143   // table switch by type
3144   if(fast_version) {
3145     __ add(PC, PC, AsmOperand(Rflags, lsl, log_max_block_size + Assembler::LogInstructionSize), ne);
3146   } else {
3147     __ ldr(PC, Address(PC, Rflags, lsl, LogBytesPerWord), ne);
3148   }
3149 
3150   // jump to itos/atos case
3151   __ b(Lint);
3152 #endif // AARCH64
3153 
3154   // table with addresses for slow version
3155   if (fast_version) {
3156     // nothing to do
3157   } else  {
3158     AARCH64_ONLY(__ align(wordSize));
3159     __ bind(Ltable);
3160     __ emit_address(Lbtos);
3161     __ emit_address(Lztos);
3162     __ emit_address(Lctos);
3163     __ emit_address(Lstos);
3164     __ emit_address(Litos);
3165     __ emit_address(Lltos);
3166     __ emit_address(Lftos);
3167     __ emit_address(Ldtos);
3168     __ emit_address(Latos);
3169   }
3170 
3171 #ifdef ASSERT
3172   int seq = 0;
3173 #endif
3174   // btos
3175   {
3176     assert(btos == seq++, "btos has unexpected value");
3177     FixedSizeCodeBlock btos_block(_masm, max_block_size, fast_version);
3178     __ bind(Lbtos);
3179     __ ldrsb(R0_tos, Address(Robj, Roffset));
3180     __ push(btos);
3181     // Rewrite bytecode to be faster
3182     if (!is_static && rc == may_rewrite) {
3183       patch_bytecode(Bytecodes::_fast_bgetfield, R0_tmp, Rtemp);
3184     }
3185     __ b(Done);
3186   }
3187 
3188   // ztos (same as btos for getfield)
3189   {
3190     assert(ztos == seq++, "btos has unexpected value");
3191     FixedSizeCodeBlock ztos_block(_masm, max_block_size, fast_version);
3192     __ bind(Lztos);
3193     __ ldrsb(R0_tos, Address(Robj, Roffset));
3194     __ push(ztos);
3195     // Rewrite bytecode to be faster (use btos fast getfield)
3196     if (!is_static && rc == may_rewrite) {
3197       patch_bytecode(Bytecodes::_fast_bgetfield, R0_tmp, Rtemp);
3198     }
3199     __ b(Done);
3200   }
3201 
3202   // ctos
3203   {
3204     assert(ctos == seq++, "ctos has unexpected value");
3205     FixedSizeCodeBlock ctos_block(_masm, max_block_size, fast_version);
3206     __ bind(Lctos);
3207     __ ldrh(R0_tos, Address(Robj, Roffset));
3208     __ push(ctos);
3209     if (!is_static && rc == may_rewrite) {
3210       patch_bytecode(Bytecodes::_fast_cgetfield, R0_tmp, Rtemp);
3211     }
3212     __ b(Done);
3213   }
3214 
3215   // stos
3216   {
3217     assert(stos == seq++, "stos has unexpected value");
3218     FixedSizeCodeBlock stos_block(_masm, max_block_size, fast_version);
3219     __ bind(Lstos);
3220     __ ldrsh(R0_tos, Address(Robj, Roffset));
3221     __ push(stos);
3222     if (!is_static && rc == may_rewrite) {
3223       patch_bytecode(Bytecodes::_fast_sgetfield, R0_tmp, Rtemp);
3224     }
3225     __ b(Done);
3226   }
3227 
3228   // itos
3229   {
3230     assert(itos == seq++, "itos has unexpected value");
3231     FixedSizeCodeBlock itos_block(_masm, max_block_size, fast_version);
3232     __ bind(Litos);
3233     __ b(shouldNotReachHere);
3234   }
3235 
3236   // ltos
3237   {
3238     assert(ltos == seq++, "ltos has unexpected value");
3239     FixedSizeCodeBlock ltos_block(_masm, max_block_size, fast_version);
3240     __ bind(Lltos);
3241 #ifdef AARCH64
3242     __ ldr(R0_tos, Address(Robj, Roffset));
3243 #else
3244     __ add(Roffset, Robj, Roffset);
3245     __ ldmia(Roffset, RegisterSet(R0_tos_lo, R1_tos_hi));
3246 #endif // AARCH64
3247     __ push(ltos);
3248     if (!is_static && rc == may_rewrite) {
3249       patch_bytecode(Bytecodes::_fast_lgetfield, R0_tmp, Rtemp);
3250     }
3251     __ b(Done);
3252   }
3253 
3254   // ftos
3255   {
3256     assert(ftos == seq++, "ftos has unexpected value");
3257     FixedSizeCodeBlock ftos_block(_masm, max_block_size, fast_version);
3258     __ bind(Lftos);
3259     // floats and ints are placed on stack in same way, so
3260     // we can use push(itos) to transfer value without using VFP
3261     __ ldr_u32(R0_tos, Address(Robj, Roffset));
3262     __ push(itos);
3263     if (!is_static && rc == may_rewrite) {
3264       patch_bytecode(Bytecodes::_fast_fgetfield, R0_tmp, Rtemp);
3265     }
3266     __ b(Done);
3267   }
3268 
3269   // dtos
3270   {
3271     assert(dtos == seq++, "dtos has unexpected value");
3272     FixedSizeCodeBlock dtos_block(_masm, max_block_size, fast_version);
3273     __ bind(Ldtos);
3274     // doubles and longs are placed on stack in the same way, so
3275     // we can use push(ltos) to transfer value without using VFP
3276 #ifdef AARCH64
3277     __ ldr(R0_tos, Address(Robj, Roffset));
3278 #else
3279     __ add(Rtemp, Robj, Roffset);
3280     __ ldmia(Rtemp, RegisterSet(R0_tos_lo, R1_tos_hi));
3281 #endif // AARCH64
3282     __ push(ltos);
3283     if (!is_static && rc == may_rewrite) {
3284       patch_bytecode(Bytecodes::_fast_dgetfield, R0_tmp, Rtemp);
3285     }
3286     __ b(Done);
3287   }
3288 
3289   // atos
3290   {
3291     assert(atos == seq++, "atos has unexpected value");
3292 
3293     // atos case for AArch64 and slow version on 32-bit ARM
3294     if(!atos_merged_with_itos) {
3295       __ bind(Latos);
3296       __ load_heap_oop(R0_tos, Address(Robj, Roffset));
3297       __ push(atos);
3298       // Rewrite bytecode to be faster
3299       if (!is_static && rc == may_rewrite) {
3300         patch_bytecode(Bytecodes::_fast_agetfield, R0_tmp, Rtemp);
3301       }
3302       __ b(Done);
3303     }
3304   }
3305 
3306   assert(vtos == seq++, "vtos has unexpected value");
3307 
3308   __ bind(shouldNotReachHere);
3309   __ should_not_reach_here();
3310 
3311   // itos and atos cases are frequent so it makes sense to move them out of table switch
3312   // atos case can be merged with itos case (and thus moved out of table switch) on 32-bit ARM, fast version only
3313 
3314   __ bind(Lint);
3315   __ ldr_s32(R0_tos, Address(Robj, Roffset));
3316   __ push(itos);
3317   // Rewrite bytecode to be faster
3318   if (!is_static && rc == may_rewrite) {
3319     patch_bytecode(Bytecodes::_fast_igetfield, R0_tmp, Rtemp);
3320   }
3321 
3322   __ bind(Done);
3323 
3324   if (gen_volatile_check) {
3325     // Check for volatile field
3326     Label notVolatile;
3327     __ tbz(Rflagsav, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3328 
3329     volatile_barrier(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp);
3330 
3331     __ bind(notVolatile);
3332   }
3333 
3334 }
3335 
3336 void TemplateTable::getfield(int byte_no) {
3337   getfield_or_static(byte_no, false);
3338 }
3339 
3340 void TemplateTable::nofast_getfield(int byte_no) {
3341   getfield_or_static(byte_no, false, may_not_rewrite);
3342 }
3343 
3344 void TemplateTable::getstatic(int byte_no) {
3345   getfield_or_static(byte_no, true);
3346 }
3347 
3348 
3349 // The registers cache and index expected to be set before call, and should not be R1 or Rtemp.
3350 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR,
3351 // except cache and index registers which are preserved.
3352 void TemplateTable::jvmti_post_field_mod(Register Rcache, Register Rindex, bool is_static) {
3353   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
3354   assert_different_registers(Rcache, Rindex, R1, Rtemp);
3355 
3356   if (__ can_post_field_modification()) {
3357     // Check to see if a field modification watch has been set before we take
3358     // the time to call into the VM.
3359     Label Lcontinue;
3360 
3361     __ ldr_global_s32(Rtemp, (address)JvmtiExport::get_field_modification_count_addr());
3362     __ cbz(Rtemp, Lcontinue);
3363 
3364     if (is_static) {
3365       // Life is simple.  Null out the object pointer.
3366       __ mov(R1, 0);
3367     } else {
3368       // Life is harder. The stack holds the value on top, followed by the object.
3369       // We don't know the size of the value, though; it could be one or two words
3370       // depending on its type. As a result, we must find the type to determine where
3371       // the object is.
3372 
3373       __ add(Rtemp, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
3374       __ ldr_u32(Rtemp, Address(Rtemp, cp_base_offset + ConstantPoolCacheEntry::flags_offset()));
3375 
3376       __ logical_shift_right(Rtemp, Rtemp, ConstantPoolCacheEntry::tos_state_shift);
3377       // Make sure we don't need to mask Rtemp after the above shift
3378       ConstantPoolCacheEntry::verify_tos_state_shift();
3379 
3380       __ cmp(Rtemp, ltos);
3381       __ cond_cmp(Rtemp, dtos, ne);
3382 #ifdef AARCH64
3383       __ mov(Rtemp, Interpreter::expr_offset_in_bytes(2));
3384       __ mov(R1, Interpreter::expr_offset_in_bytes(1));
3385       __ mov(R1, Rtemp, eq);
3386       __ ldr(R1, Address(Rstack_top, R1));
3387 #else
3388       // two word value (ltos/dtos)
3389       __ ldr(R1, Address(SP, Interpreter::expr_offset_in_bytes(2)), eq);
3390 
3391       // one word value (not ltos, dtos)
3392       __ ldr(R1, Address(SP, Interpreter::expr_offset_in_bytes(1)), ne);
3393 #endif // AARCH64
3394     }
3395 
3396     // cache entry pointer
3397     __ add(R2, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
3398     __ add(R2, R2, in_bytes(cp_base_offset));
3399 
3400     // object (tos)
3401     __ mov(R3, Rstack_top);
3402 
3403     // R1: object pointer set up above (NULL if static)
3404     // R2: cache entry pointer
3405     // R3: value object on the stack
3406     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification),
3407                R1, R2, R3);
3408     __ get_cache_and_index_at_bcp(Rcache, Rindex, 1);
3409 
3410     __ bind(Lcontinue);
3411   }
3412 }
3413 
3414 
3415 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
3416   transition(vtos, vtos);
3417 
3418   const Register Roffset  = R2_tmp;
3419   const Register Robj     = R3_tmp;
3420   const Register Rcache   = R4_tmp;
3421   const Register Rflagsav = Rtmp_save0;  // R4/R19
3422   const Register Rindex   = R5_tmp;
3423   const Register Rflags   = R5_tmp;
3424 
3425   const bool gen_volatile_check = os::is_MP();
3426 
3427   resolve_cache_and_index(byte_no, Rcache, Rindex, sizeof(u2));
3428   jvmti_post_field_mod(Rcache, Rindex, is_static);
3429   load_field_cp_cache_entry(Rcache, Rindex, Roffset, Rflags, Robj, is_static);
3430 
3431   if (gen_volatile_check) {
3432     // Check for volatile field
3433     Label notVolatile;
3434     __ mov(Rflagsav, Rflags);
3435     __ tbz(Rflagsav, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3436 
3437     volatile_barrier(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
3438 
3439     __ bind(notVolatile);
3440   }
3441 
3442   Label Done, Lint, shouldNotReachHere;
3443   Label Ltable, Lbtos, Lztos, Lctos, Lstos, Litos, Lltos, Lftos, Ldtos, Latos;
3444 
3445   // compute type
3446   __ logical_shift_right(Rflags, Rflags, ConstantPoolCacheEntry::tos_state_shift);
3447   // Make sure we don't need to mask flags after the above shift
3448   ConstantPoolCacheEntry::verify_tos_state_shift();
3449 
3450   // There are actually two versions of implementation of putfield/putstatic:
3451   //
3452   // 32-bit ARM:
3453   // 1) Table switch using add(PC,...) instruction (fast_version)
3454   // 2) Table switch using ldr(PC,...) instruction
3455   //
3456   // AArch64:
3457   // 1) Table switch using adr/add/br instructions (fast_version)
3458   // 2) Table switch using adr/ldr/br instructions
3459   //
3460   // First version requires fixed size of code block for each case and
3461   // can not be used in RewriteBytecodes and VerifyOops
3462   // modes.
3463 
3464   // Size of fixed size code block for fast_version (in instructions)
3465   const int log_max_block_size = AARCH64_ONLY(is_static ? 2 : 3) NOT_AARCH64(3);
3466   const int max_block_size = 1 << log_max_block_size;
3467 
3468   // Decide if fast version is enabled
3469   bool fast_version = (is_static || !RewriteBytecodes) && !VerifyOops && !ZapHighNonSignificantBits;
3470 
3471   assert(number_of_states == 10, "number of tos states should be equal to 9");
3472 
3473   // itos case is frequent and is moved outside table switch
3474   __ cmp(Rflags, itos);
3475 
3476 #ifdef AARCH64
3477   __ b(Lint, eq);
3478 
3479   if (fast_version) {
3480     __ adr(Rtemp, Lbtos);
3481     __ add(Rtemp, Rtemp, AsmOperand(Rflags, lsl, log_max_block_size + Assembler::LogInstructionSize));
3482     __ br(Rtemp);
3483   } else {
3484     __ adr(Rtemp, Ltable);
3485     __ ldr(Rtemp, Address::indexed_ptr(Rtemp, Rflags));
3486     __ br(Rtemp);
3487   }
3488 #else
3489   // table switch by type
3490   if (fast_version) {
3491     __ add(PC, PC, AsmOperand(Rflags, lsl, log_max_block_size + Assembler::LogInstructionSize), ne);
3492   } else  {
3493     __ ldr(PC, Address(PC, Rflags, lsl, LogBytesPerWord), ne);
3494   }
3495 
3496   // jump to itos case
3497   __ b(Lint);
3498 #endif // AARCH64
3499 
3500   // table with addresses for slow version
3501   if (fast_version) {
3502     // nothing to do
3503   } else  {
3504     AARCH64_ONLY(__ align(wordSize));
3505     __ bind(Ltable);
3506     __ emit_address(Lbtos);
3507     __ emit_address(Lztos);
3508     __ emit_address(Lctos);
3509     __ emit_address(Lstos);
3510     __ emit_address(Litos);
3511     __ emit_address(Lltos);
3512     __ emit_address(Lftos);
3513     __ emit_address(Ldtos);
3514     __ emit_address(Latos);
3515   }
3516 
3517 #ifdef ASSERT
3518   int seq = 0;
3519 #endif
3520   // btos
3521   {
3522     assert(btos == seq++, "btos has unexpected value");
3523     FixedSizeCodeBlock btos_block(_masm, max_block_size, fast_version);
3524     __ bind(Lbtos);
3525     __ pop(btos);
3526     if (!is_static) pop_and_check_object(Robj);
3527     __ strb(R0_tos, Address(Robj, Roffset));
3528     if (!is_static && rc == may_rewrite) {
3529       patch_bytecode(Bytecodes::_fast_bputfield, R0_tmp, Rtemp, true, byte_no);
3530     }
3531     __ b(Done);
3532   }
3533 
3534   // ztos
3535   {
3536     assert(ztos == seq++, "ztos has unexpected value");
3537     FixedSizeCodeBlock ztos_block(_masm, max_block_size, fast_version);
3538     __ bind(Lztos);
3539     __ pop(ztos);
3540     if (!is_static) pop_and_check_object(Robj);
3541     __ and_32(R0_tos, R0_tos, 1);
3542     __ strb(R0_tos, Address(Robj, Roffset));
3543     if (!is_static && rc == may_rewrite) {
3544       patch_bytecode(Bytecodes::_fast_zputfield, R0_tmp, Rtemp, true, byte_no);
3545     }
3546     __ b(Done);
3547   }
3548 
3549   // ctos
3550   {
3551     assert(ctos == seq++, "ctos has unexpected value");
3552     FixedSizeCodeBlock ctos_block(_masm, max_block_size, fast_version);
3553     __ bind(Lctos);
3554     __ pop(ctos);
3555     if (!is_static) pop_and_check_object(Robj);
3556     __ strh(R0_tos, Address(Robj, Roffset));
3557     if (!is_static && rc == may_rewrite) {
3558       patch_bytecode(Bytecodes::_fast_cputfield, R0_tmp, Rtemp, true, byte_no);
3559     }
3560     __ b(Done);
3561   }
3562 
3563   // stos
3564   {
3565     assert(stos == seq++, "stos has unexpected value");
3566     FixedSizeCodeBlock stos_block(_masm, max_block_size, fast_version);
3567     __ bind(Lstos);
3568     __ pop(stos);
3569     if (!is_static) pop_and_check_object(Robj);
3570     __ strh(R0_tos, Address(Robj, Roffset));
3571     if (!is_static && rc == may_rewrite) {
3572       patch_bytecode(Bytecodes::_fast_sputfield, R0_tmp, Rtemp, true, byte_no);
3573     }
3574     __ b(Done);
3575   }
3576 
3577   // itos
3578   {
3579     assert(itos == seq++, "itos has unexpected value");
3580     FixedSizeCodeBlock itos_block(_masm, max_block_size, fast_version);
3581     __ bind(Litos);
3582     __ b(shouldNotReachHere);
3583   }
3584 
3585   // ltos
3586   {
3587     assert(ltos == seq++, "ltos has unexpected value");
3588     FixedSizeCodeBlock ltos_block(_masm, max_block_size, fast_version);
3589     __ bind(Lltos);
3590     __ pop(ltos);
3591     if (!is_static) pop_and_check_object(Robj);
3592 #ifdef AARCH64
3593     __ str(R0_tos, Address(Robj, Roffset));
3594 #else
3595     __ add(Roffset, Robj, Roffset);
3596     __ stmia(Roffset, RegisterSet(R0_tos_lo, R1_tos_hi));
3597 #endif // AARCH64
3598     if (!is_static && rc == may_rewrite) {
3599       patch_bytecode(Bytecodes::_fast_lputfield, R0_tmp, Rtemp, true, byte_no);
3600     }
3601     __ b(Done);
3602   }
3603 
3604   // ftos
3605   {
3606     assert(ftos == seq++, "ftos has unexpected value");
3607     FixedSizeCodeBlock ftos_block(_masm, max_block_size, fast_version);
3608     __ bind(Lftos);
3609     // floats and ints are placed on stack in the same way, so
3610     // we can use pop(itos) to transfer value without using VFP
3611     __ pop(itos);
3612     if (!is_static) pop_and_check_object(Robj);
3613     __ str_32(R0_tos, Address(Robj, Roffset));
3614     if (!is_static && rc == may_rewrite) {
3615       patch_bytecode(Bytecodes::_fast_fputfield, R0_tmp, Rtemp, true, byte_no);
3616     }
3617     __ b(Done);
3618   }
3619 
3620   // dtos
3621   {
3622     assert(dtos == seq++, "dtos has unexpected value");
3623     FixedSizeCodeBlock dtos_block(_masm, max_block_size, fast_version);
3624     __ bind(Ldtos);
3625     // doubles and longs are placed on stack in the same way, so
3626     // we can use pop(ltos) to transfer value without using VFP
3627     __ pop(ltos);
3628     if (!is_static) pop_and_check_object(Robj);
3629 #ifdef AARCH64
3630     __ str(R0_tos, Address(Robj, Roffset));
3631 #else
3632     __ add(Rtemp, Robj, Roffset);
3633     __ stmia(Rtemp, RegisterSet(R0_tos_lo, R1_tos_hi));
3634 #endif // AARCH64
3635     if (!is_static && rc == may_rewrite) {
3636       patch_bytecode(Bytecodes::_fast_dputfield, R0_tmp, Rtemp, true, byte_no);
3637     }
3638     __ b(Done);
3639   }
3640 
3641   // atos
3642   {
3643     assert(atos == seq++, "dtos has unexpected value");
3644     __ bind(Latos);
3645     __ pop(atos);
3646     if (!is_static) pop_and_check_object(Robj);
3647     // Store into the field
3648     do_oop_store(_masm, Address(Robj, Roffset), R0_tos, Rtemp, R1_tmp, R5_tmp, _bs->kind(), false, false);
3649     if (!is_static && rc == may_rewrite) {
3650       patch_bytecode(Bytecodes::_fast_aputfield, R0_tmp, Rtemp, true, byte_no);
3651     }
3652     __ b(Done);
3653   }
3654 
3655   __ bind(shouldNotReachHere);
3656   __ should_not_reach_here();
3657 
3658   // itos case is frequent and is moved outside table switch
3659   __ bind(Lint);
3660   __ pop(itos);
3661   if (!is_static) pop_and_check_object(Robj);
3662   __ str_32(R0_tos, Address(Robj, Roffset));
3663   if (!is_static && rc == may_rewrite) {
3664     patch_bytecode(Bytecodes::_fast_iputfield, R0_tmp, Rtemp, true, byte_no);
3665   }
3666 
3667   __ bind(Done);
3668 
3669   if (gen_volatile_check) {
3670     Label notVolatile;
3671     if (is_static) {
3672       // Just check for volatile. Memory barrier for static final field
3673       // is handled by class initialization.
3674       __ tbz(Rflagsav, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3675       volatile_barrier(MacroAssembler::StoreLoad, Rtemp);
3676       __ bind(notVolatile);
3677     } else {
3678       // Check for volatile field and final field
3679       Label skipMembar;
3680 
3681       __ tst(Rflagsav, 1 << ConstantPoolCacheEntry::is_volatile_shift |
3682                        1 << ConstantPoolCacheEntry::is_final_shift);
3683       __ b(skipMembar, eq);
3684 
3685       __ tbz(Rflagsav, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3686 
3687       // StoreLoad barrier after volatile field write
3688       volatile_barrier(MacroAssembler::StoreLoad, Rtemp);
3689       __ b(skipMembar);
3690 
3691       // StoreStore barrier after final field write
3692       __ bind(notVolatile);
3693       volatile_barrier(MacroAssembler::StoreStore, Rtemp);
3694 
3695       __ bind(skipMembar);
3696     }
3697   }
3698 
3699 }
3700 
3701 void TemplateTable::putfield(int byte_no) {
3702   putfield_or_static(byte_no, false);
3703 }
3704 
3705 void TemplateTable::nofast_putfield(int byte_no) {
3706   putfield_or_static(byte_no, false, may_not_rewrite);
3707 }
3708 
3709 void TemplateTable::putstatic(int byte_no) {
3710   putfield_or_static(byte_no, true);
3711 }
3712 
3713 
3714 void TemplateTable::jvmti_post_fast_field_mod() {
3715   // This version of jvmti_post_fast_field_mod() is not used on ARM
3716   Unimplemented();
3717 }
3718 
3719 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR,
3720 // but preserves tosca with the given state.
3721 void TemplateTable::jvmti_post_fast_field_mod(TosState state) {
3722   if (__ can_post_field_modification()) {
3723     // Check to see if a field modification watch has been set before we take
3724     // the time to call into the VM.
3725     Label done;
3726 
3727     __ ldr_global_s32(R2, (address)JvmtiExport::get_field_modification_count_addr());
3728     __ cbz(R2, done);
3729 
3730     __ pop_ptr(R3);               // copy the object pointer from tos
3731     __ verify_oop(R3);
3732     __ push_ptr(R3);              // put the object pointer back on tos
3733 
3734     __ push(state);               // save value on the stack
3735 
3736     // access constant pool cache entry
3737     __ get_cache_entry_pointer_at_bcp(R2, R1, 1);
3738 
3739     __ mov(R1, R3);
3740     assert(Interpreter::expr_offset_in_bytes(0) == 0, "adjust this code");
3741     __ mov(R3, Rstack_top); // put tos addr into R3
3742 
3743     // R1: object pointer copied above
3744     // R2: cache entry pointer
3745     // R3: jvalue object on the stack
3746     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), R1, R2, R3);
3747 
3748     __ pop(state);                // restore value
3749 
3750     __ bind(done);
3751   }
3752 }
3753 
3754 
3755 void TemplateTable::fast_storefield(TosState state) {
3756   transition(state, vtos);
3757 
3758   ByteSize base = ConstantPoolCache::base_offset();
3759 
3760   jvmti_post_fast_field_mod(state);
3761 
3762   const Register Rcache  = R2_tmp;
3763   const Register Rindex  = R3_tmp;
3764   const Register Roffset = R3_tmp;
3765   const Register Rflags  = Rtmp_save0; // R4/R19
3766   const Register Robj    = R5_tmp;
3767 
3768   const bool gen_volatile_check = os::is_MP();
3769 
3770   // access constant pool cache
3771   __ get_cache_and_index_at_bcp(Rcache, Rindex, 1);
3772 
3773   __ add(Rcache, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
3774 
3775   if (gen_volatile_check) {
3776     // load flags to test volatile
3777     __ ldr_u32(Rflags, Address(Rcache, base + ConstantPoolCacheEntry::flags_offset()));
3778   }
3779 
3780   // replace index with field offset from cache entry
3781   __ ldr(Roffset, Address(Rcache, base + ConstantPoolCacheEntry::f2_offset()));
3782 
3783   if (gen_volatile_check) {
3784     // Check for volatile store
3785     Label notVolatile;
3786     __ tbz(Rflags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3787 
3788     // TODO-AARCH64 on AArch64, store-release instructions can be used to get rid of this explict barrier
3789     volatile_barrier(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
3790 
3791     __ bind(notVolatile);
3792   }
3793 
3794   // Get object from stack
3795   pop_and_check_object(Robj);
3796 
3797   // access field
3798   switch (bytecode()) {
3799     case Bytecodes::_fast_zputfield: __ and_32(R0_tos, R0_tos, 1);
3800                                      // fall through
3801     case Bytecodes::_fast_bputfield: __ strb(R0_tos, Address(Robj, Roffset)); break;
3802     case Bytecodes::_fast_sputfield: // fall through
3803     case Bytecodes::_fast_cputfield: __ strh(R0_tos, Address(Robj, Roffset)); break;
3804     case Bytecodes::_fast_iputfield: __ str_32(R0_tos, Address(Robj, Roffset)); break;
3805 #ifdef AARCH64
3806     case Bytecodes::_fast_lputfield: __ str  (R0_tos, Address(Robj, Roffset)); break;
3807     case Bytecodes::_fast_fputfield: __ str_s(S0_tos, Address(Robj, Roffset)); break;
3808     case Bytecodes::_fast_dputfield: __ str_d(D0_tos, Address(Robj, Roffset)); break;
3809 #else
3810     case Bytecodes::_fast_lputfield: __ add(Robj, Robj, Roffset);
3811                                      __ stmia(Robj, RegisterSet(R0_tos_lo, R1_tos_hi)); break;
3812 
3813 #ifdef __SOFTFP__
3814     case Bytecodes::_fast_fputfield: __ str(R0_tos, Address(Robj, Roffset));  break;
3815     case Bytecodes::_fast_dputfield: __ add(Robj, Robj, Roffset);
3816                                      __ stmia(Robj, RegisterSet(R0_tos_lo, R1_tos_hi)); break;
3817 #else
3818     case Bytecodes::_fast_fputfield: __ add(Robj, Robj, Roffset);
3819                                      __ fsts(S0_tos, Address(Robj));          break;
3820     case Bytecodes::_fast_dputfield: __ add(Robj, Robj, Roffset);
3821                                      __ fstd(D0_tos, Address(Robj));          break;
3822 #endif // __SOFTFP__
3823 #endif // AARCH64
3824 
3825     case Bytecodes::_fast_aputfield:
3826       do_oop_store(_masm, Address(Robj, Roffset), R0_tos, Rtemp, R1_tmp, R2_tmp, _bs->kind(), false, false);
3827       break;
3828 
3829     default:
3830       ShouldNotReachHere();
3831   }
3832 
3833   if (gen_volatile_check) {
3834     Label notVolatile;
3835     Label skipMembar;
3836     __ tst(Rflags, 1 << ConstantPoolCacheEntry::is_volatile_shift |
3837                    1 << ConstantPoolCacheEntry::is_final_shift);
3838     __ b(skipMembar, eq);
3839 
3840     __ tbz(Rflags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3841 
3842     // StoreLoad barrier after volatile field write
3843     volatile_barrier(MacroAssembler::StoreLoad, Rtemp);
3844     __ b(skipMembar);
3845 
3846     // StoreStore barrier after final field write
3847     __ bind(notVolatile);
3848     volatile_barrier(MacroAssembler::StoreStore, Rtemp);
3849 
3850     __ bind(skipMembar);
3851   }
3852 }
3853 
3854 
3855 void TemplateTable::fast_accessfield(TosState state) {
3856   transition(atos, state);
3857 
3858   // do the JVMTI work here to avoid disturbing the register state below
3859   if (__ can_post_field_access()) {
3860     // Check to see if a field access watch has been set before we take
3861     // the time to call into the VM.
3862     Label done;
3863     __ ldr_global_s32(R2, (address) JvmtiExport::get_field_access_count_addr());
3864     __ cbz(R2, done);
3865     // access constant pool cache entry
3866     __ get_cache_entry_pointer_at_bcp(R2, R1, 1);
3867     __ push_ptr(R0_tos);  // save object pointer before call_VM() clobbers it
3868     __ verify_oop(R0_tos);
3869     __ mov(R1, R0_tos);
3870     // R1: object pointer copied above
3871     // R2: cache entry pointer
3872     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access), R1, R2);
3873     __ pop_ptr(R0_tos);   // restore object pointer
3874 
3875     __ bind(done);
3876   }
3877 
3878   const Register Robj    = R0_tos;
3879   const Register Rcache  = R2_tmp;
3880   const Register Rflags  = R2_tmp;
3881   const Register Rindex  = R3_tmp;
3882   const Register Roffset = R3_tmp;
3883 
3884   const bool gen_volatile_check = os::is_MP();
3885 
3886   // access constant pool cache
3887   __ get_cache_and_index_at_bcp(Rcache, Rindex, 1);
3888   // replace index with field offset from cache entry
3889   __ add(Rtemp, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
3890   __ ldr(Roffset, Address(Rtemp, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset()));
3891 
3892   if (gen_volatile_check) {
3893     // load flags to test volatile
3894     __ ldr_u32(Rflags, Address(Rtemp, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
3895   }
3896 
3897   __ verify_oop(Robj);
3898   __ null_check(Robj, Rtemp);
3899 
3900   // access field
3901   switch (bytecode()) {
3902     case Bytecodes::_fast_bgetfield: __ ldrsb(R0_tos, Address(Robj, Roffset)); break;
3903     case Bytecodes::_fast_sgetfield: __ ldrsh(R0_tos, Address(Robj, Roffset)); break;
3904     case Bytecodes::_fast_cgetfield: __ ldrh (R0_tos, Address(Robj, Roffset)); break;
3905     case Bytecodes::_fast_igetfield: __ ldr_s32(R0_tos, Address(Robj, Roffset)); break;
3906 #ifdef AARCH64
3907     case Bytecodes::_fast_lgetfield: __ ldr  (R0_tos, Address(Robj, Roffset)); break;
3908     case Bytecodes::_fast_fgetfield: __ ldr_s(S0_tos, Address(Robj, Roffset)); break;
3909     case Bytecodes::_fast_dgetfield: __ ldr_d(D0_tos, Address(Robj, Roffset)); break;
3910 #else
3911     case Bytecodes::_fast_lgetfield: __ add(Roffset, Robj, Roffset);
3912                                      __ ldmia(Roffset, RegisterSet(R0_tos_lo, R1_tos_hi)); break;
3913 #ifdef __SOFTFP__
3914     case Bytecodes::_fast_fgetfield: __ ldr  (R0_tos, Address(Robj, Roffset)); break;
3915     case Bytecodes::_fast_dgetfield: __ add(Roffset, Robj, Roffset);
3916                                      __ ldmia(Roffset, RegisterSet(R0_tos_lo, R1_tos_hi)); break;
3917 #else
3918     case Bytecodes::_fast_fgetfield: __ add(Roffset, Robj, Roffset); __ flds(S0_tos, Address(Roffset)); break;
3919     case Bytecodes::_fast_dgetfield: __ add(Roffset, Robj, Roffset); __ fldd(D0_tos, Address(Roffset)); break;
3920 #endif // __SOFTFP__
3921 #endif // AARCH64
3922     case Bytecodes::_fast_agetfield: __ load_heap_oop(R0_tos, Address(Robj, Roffset)); __ verify_oop(R0_tos); break;
3923     default:
3924       ShouldNotReachHere();
3925   }
3926 
3927   if (gen_volatile_check) {
3928     // Check for volatile load
3929     Label notVolatile;
3930     __ tbz(Rflags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3931 
3932     // TODO-AARCH64 on AArch64, load-acquire instructions can be used to get rid of this explict barrier
3933     volatile_barrier(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp);
3934 
3935     __ bind(notVolatile);
3936   }
3937 }
3938 
3939 
3940 void TemplateTable::fast_xaccess(TosState state) {
3941   transition(vtos, state);
3942 
3943   const Register Robj = R1_tmp;
3944   const Register Rcache = R2_tmp;
3945   const Register Rindex = R3_tmp;
3946   const Register Roffset = R3_tmp;
3947   const Register Rflags = R4_tmp;
3948   Label done;
3949 
3950   // get receiver
3951   __ ldr(Robj, aaddress(0));
3952 
3953   // access constant pool cache
3954   __ get_cache_and_index_at_bcp(Rcache, Rindex, 2);
3955   __ add(Rtemp, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
3956   __ ldr(Roffset, Address(Rtemp, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset()));
3957 
3958   const bool gen_volatile_check = os::is_MP();
3959 
3960   if (gen_volatile_check) {
3961     // load flags to test volatile
3962     __ ldr_u32(Rflags, Address(Rtemp, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
3963   }
3964 
3965   // make sure exception is reported in correct bcp range (getfield is next instruction)
3966   __ add(Rbcp, Rbcp, 1);
3967   __ null_check(Robj, Rtemp);
3968   __ sub(Rbcp, Rbcp, 1);
3969 
3970 #ifdef AARCH64
3971   if (gen_volatile_check) {
3972     Label notVolatile;
3973     __ tbz(Rflags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3974 
3975     __ add(Rtemp, Robj, Roffset);
3976 
3977     if (state == itos) {
3978       __ ldar_w(R0_tos, Rtemp);
3979     } else if (state == atos) {
3980       if (UseCompressedOops) {
3981         __ ldar_w(R0_tos, Rtemp);
3982         __ decode_heap_oop(R0_tos);
3983       } else {
3984         __ ldar(R0_tos, Rtemp);
3985       }
3986       __ verify_oop(R0_tos);
3987     } else if (state == ftos) {
3988       __ ldar_w(R0_tos, Rtemp);
3989       __ fmov_sw(S0_tos, R0_tos);
3990     } else {
3991       ShouldNotReachHere();
3992     }
3993     __ b(done);
3994 
3995     __ bind(notVolatile);
3996   }
3997 #endif // AARCH64
3998 
3999   if (state == itos) {
4000     __ ldr_s32(R0_tos, Address(Robj, Roffset));
4001   } else if (state == atos) {
4002     __ load_heap_oop(R0_tos, Address(Robj, Roffset));
4003     __ verify_oop(R0_tos);
4004   } else if (state == ftos) {
4005 #ifdef AARCH64
4006     __ ldr_s(S0_tos, Address(Robj, Roffset));
4007 #else
4008 #ifdef __SOFTFP__
4009     __ ldr(R0_tos, Address(Robj, Roffset));
4010 #else
4011     __ add(Roffset, Robj, Roffset);
4012     __ flds(S0_tos, Address(Roffset));
4013 #endif // __SOFTFP__
4014 #endif // AARCH64
4015   } else {
4016     ShouldNotReachHere();
4017   }
4018 
4019 #ifndef AARCH64
4020   if (gen_volatile_check) {
4021     // Check for volatile load
4022     Label notVolatile;
4023     __ tbz(Rflags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
4024 
4025     volatile_barrier(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp);
4026 
4027     __ bind(notVolatile);
4028   }
4029 #endif // !AARCH64
4030 
4031   __ bind(done);
4032 }
4033 
4034 
4035 
4036 //----------------------------------------------------------------------------------------------------
4037 // Calls
4038 
4039 void TemplateTable::count_calls(Register method, Register temp) {
4040   // implemented elsewhere
4041   ShouldNotReachHere();
4042 }
4043 
4044 
4045 void TemplateTable::prepare_invoke(int byte_no,
4046                                    Register method,  // linked method (or i-klass)
4047                                    Register index,   // itable index, MethodType, etc.
4048                                    Register recv,    // if caller wants to see it
4049                                    Register flags    // if caller wants to test it
4050                                    ) {
4051   // determine flags
4052   const Bytecodes::Code code = bytecode();
4053   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
4054   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
4055   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
4056   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
4057   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
4058   const bool load_receiver       = (recv != noreg);
4059   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
4060   assert(recv  == noreg || recv  == R2, "");
4061   assert(flags == noreg || flags == R3, "");
4062 
4063   // setup registers & access constant pool cache
4064   if (recv  == noreg)  recv  = R2;
4065   if (flags == noreg)  flags = R3;
4066   const Register temp = Rtemp;
4067   const Register ret_type = R1_tmp;
4068   assert_different_registers(method, index, flags, recv, LR, ret_type, temp);
4069 
4070   // save 'interpreter return address'
4071   __ save_bcp();
4072 
4073   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
4074 
4075   // maybe push extra argument
4076   if (is_invokedynamic || is_invokehandle) {
4077     Label L_no_push;
4078     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
4079     __ mov(temp, index);
4080     assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
4081     __ load_resolved_reference_at_index(index, temp);
4082     __ verify_oop(index);
4083     __ push_ptr(index);  // push appendix (MethodType, CallSite, etc.)
4084     __ bind(L_no_push);
4085   }
4086 
4087   // load receiver if needed (after extra argument is pushed so parameter size is correct)
4088   if (load_receiver) {
4089     __ andr(temp, flags, (uintx)ConstantPoolCacheEntry::parameter_size_mask);  // get parameter size
4090     Address recv_addr = __ receiver_argument_address(Rstack_top, temp, recv);
4091     __ ldr(recv, recv_addr);
4092     __ verify_oop(recv);
4093   }
4094 
4095   // compute return type
4096   __ logical_shift_right(ret_type, flags, ConstantPoolCacheEntry::tos_state_shift);
4097   // Make sure we don't need to mask flags after the above shift
4098   ConstantPoolCacheEntry::verify_tos_state_shift();
4099   // load return address
4100   { const address table = (address) Interpreter::invoke_return_entry_table_for(code);
4101     __ mov_slow(temp, table);
4102     __ ldr(LR, Address::indexed_ptr(temp, ret_type));
4103   }
4104 }
4105 
4106 
4107 void TemplateTable::invokevirtual_helper(Register index,
4108                                          Register recv,
4109                                          Register flags) {
4110 
4111   const Register recv_klass = R2_tmp;
4112 
4113   assert_different_registers(index, recv, flags, Rtemp);
4114   assert_different_registers(index, recv_klass, R0_tmp, Rtemp);
4115 
4116   // Test for an invoke of a final method
4117   Label notFinal;
4118   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
4119 
4120   assert(index == Rmethod, "Method* must be Rmethod, for interpreter calling convention");
4121 
4122   // do the call - the index is actually the method to call
4123 
4124   // It's final, need a null check here!
4125   __ null_check(recv, Rtemp);
4126 
4127   // profile this call
4128   __ profile_final_call(R0_tmp);
4129 
4130   __ jump_from_interpreted(Rmethod);
4131 
4132   __ bind(notFinal);
4133 
4134   // get receiver klass
4135   __ null_check(recv, Rtemp, oopDesc::klass_offset_in_bytes());
4136   __ load_klass(recv_klass, recv);
4137 
4138   // profile this call
4139   __ profile_virtual_call(R0_tmp, recv_klass);
4140 
4141   // get target Method* & entry point
4142   const int base = in_bytes(Klass::vtable_start_offset());
4143   assert(vtableEntry::size() == 1, "adjust the scaling in the code below");
4144   __ add(Rtemp, recv_klass, AsmOperand(index, lsl, LogHeapWordSize));
4145   __ ldr(Rmethod, Address(Rtemp, base + vtableEntry::method_offset_in_bytes()));
4146   __ jump_from_interpreted(Rmethod);
4147 }
4148 
4149 void TemplateTable::invokevirtual(int byte_no) {
4150   transition(vtos, vtos);
4151   assert(byte_no == f2_byte, "use this argument");
4152 
4153   const Register Rrecv  = R2_tmp;
4154   const Register Rflags = R3_tmp;
4155 
4156   prepare_invoke(byte_no, Rmethod, noreg, Rrecv, Rflags);
4157 
4158   // Rmethod: index
4159   // Rrecv:   receiver
4160   // Rflags:  flags
4161   // LR:      return address
4162 
4163   invokevirtual_helper(Rmethod, Rrecv, Rflags);
4164 }
4165 
4166 
4167 void TemplateTable::invokespecial(int byte_no) {
4168   transition(vtos, vtos);
4169   assert(byte_no == f1_byte, "use this argument");
4170   const Register Rrecv  = R2_tmp;
4171   prepare_invoke(byte_no, Rmethod, noreg, Rrecv);
4172   __ verify_oop(Rrecv);
4173   __ null_check(Rrecv, Rtemp);
4174   // do the call
4175   __ profile_call(Rrecv);
4176   __ jump_from_interpreted(Rmethod);
4177 }
4178 
4179 
4180 void TemplateTable::invokestatic(int byte_no) {
4181   transition(vtos, vtos);
4182   assert(byte_no == f1_byte, "use this argument");
4183   prepare_invoke(byte_no, Rmethod);
4184   // do the call
4185   __ profile_call(R2_tmp);
4186   __ jump_from_interpreted(Rmethod);
4187 }
4188 
4189 
4190 void TemplateTable::fast_invokevfinal(int byte_no) {
4191   transition(vtos, vtos);
4192   assert(byte_no == f2_byte, "use this argument");
4193   __ stop("fast_invokevfinal is not used on ARM");
4194 }
4195 
4196 
4197 void TemplateTable::invokeinterface(int byte_no) {
4198   transition(vtos, vtos);
4199   assert(byte_no == f1_byte, "use this argument");
4200 
4201   const Register Ritable = R1_tmp;
4202   const Register Rrecv   = R2_tmp;
4203   const Register Rinterf = R5_tmp;
4204   const Register Rindex  = R4_tmp;
4205   const Register Rflags  = R3_tmp;
4206   const Register Rklass  = R3_tmp;
4207 
4208   prepare_invoke(byte_no, Rinterf, Rindex, Rrecv, Rflags);
4209 
4210   // Special case of invokeinterface called for virtual method of
4211   // java.lang.Object.  See cpCacheOop.cpp for details.
4212   // This code isn't produced by javac, but could be produced by
4213   // another compliant java compiler.
4214   Label notMethod;
4215   __ tbz(Rflags, ConstantPoolCacheEntry::is_forced_virtual_shift, notMethod);
4216 
4217   __ mov(Rmethod, Rindex);
4218   invokevirtual_helper(Rmethod, Rrecv, Rflags);
4219   __ bind(notMethod);
4220 
4221   // Get receiver klass into Rklass - also a null check
4222   __ load_klass(Rklass, Rrecv);
4223 
4224   // profile this call
4225   __ profile_virtual_call(R0_tmp, Rklass);
4226 
4227   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
4228   const int base = in_bytes(Klass::vtable_start_offset());
4229   assert(vtableEntry::size() == 1, "adjust the scaling in the code below");
4230   __ ldr_s32(Rtemp, Address(Rklass, Klass::vtable_length_offset())); // Get length of vtable
4231   __ add(Ritable, Rklass, base);
4232   __ add(Ritable, Ritable, AsmOperand(Rtemp, lsl, LogBytesPerWord));
4233 
4234   Label entry, search, interface_ok;
4235 
4236   __ b(entry);
4237 
4238   __ bind(search);
4239   __ add(Ritable, Ritable, itableOffsetEntry::size() * HeapWordSize);
4240 
4241   __ bind(entry);
4242 
4243   // Check that the entry is non-null.  A null entry means that the receiver
4244   // class doesn't implement the interface, and wasn't the same as the
4245   // receiver class checked when the interface was resolved.
4246 
4247   __ ldr(Rtemp, Address(Ritable, itableOffsetEntry::interface_offset_in_bytes()));
4248   __ cbnz(Rtemp, interface_ok);
4249 
4250   // throw exception
4251   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4252                    InterpreterRuntime::throw_IncompatibleClassChangeError));
4253 
4254   // the call_VM checks for exception, so we should never return here.
4255   __ should_not_reach_here();
4256 
4257   __ bind(interface_ok);
4258 
4259   __ cmp(Rinterf, Rtemp);
4260   __ b(search, ne);
4261 
4262   __ ldr_s32(Rtemp, Address(Ritable, itableOffsetEntry::offset_offset_in_bytes()));
4263   __ add(Rtemp, Rtemp, Rklass); // Add offset to Klass*
4264   assert(itableMethodEntry::size() == 1, "adjust the scaling in the code below");
4265 
4266   __ ldr(Rmethod, Address::indexed_ptr(Rtemp, Rindex));
4267 
4268   // Rmethod: Method* to call
4269 
4270   // Check for abstract method error
4271   // Note: This should be done more efficiently via a throw_abstract_method_error
4272   //       interpreter entry point and a conditional jump to it in case of a null
4273   //       method.
4274   { Label L;
4275     __ cbnz(Rmethod, L);
4276     // throw exception
4277     // note: must restore interpreter registers to canonical
4278     //       state for exception handling to work correctly!
4279     __ restore_method();
4280     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
4281     // the call_VM checks for exception, so we should never return here.
4282     __ should_not_reach_here();
4283     __ bind(L);
4284   }
4285 
4286   // do the call
4287   __ jump_from_interpreted(Rmethod);
4288 }
4289 
4290 void TemplateTable::invokehandle(int byte_no) {
4291   transition(vtos, vtos);
4292 
4293   // TODO-AARCH64 review register usage
4294   const Register Rrecv  = R2_tmp;
4295   const Register Rmtype = R4_tmp;
4296   const Register R5_method = R5_tmp;  // can't reuse Rmethod!
4297 
4298   prepare_invoke(byte_no, R5_method, Rmtype, Rrecv);
4299   __ null_check(Rrecv, Rtemp);
4300 
4301   // Rmtype:  MethodType object (from cpool->resolved_references[f1], if necessary)
4302   // Rmethod: MH.invokeExact_MT method (from f2)
4303 
4304   // Note:  Rmtype is already pushed (if necessary) by prepare_invoke
4305 
4306   // do the call
4307   __ profile_final_call(R3_tmp);  // FIXME: profile the LambdaForm also
4308   __ mov(Rmethod, R5_method);
4309   __ jump_from_interpreted(Rmethod);
4310 }
4311 
4312 void TemplateTable::invokedynamic(int byte_no) {
4313   transition(vtos, vtos);
4314 
4315   // TODO-AARCH64 review register usage
4316   const Register Rcallsite = R4_tmp;
4317   const Register R5_method = R5_tmp;  // can't reuse Rmethod!
4318 
4319   prepare_invoke(byte_no, R5_method, Rcallsite);
4320 
4321   // Rcallsite: CallSite object (from cpool->resolved_references[f1])
4322   // Rmethod:   MH.linkToCallSite method (from f2)
4323 
4324   // Note:  Rcallsite is already pushed by prepare_invoke
4325 
4326   if (ProfileInterpreter) {
4327     __ profile_call(R2_tmp);
4328   }
4329 
4330   // do the call
4331   __ mov(Rmethod, R5_method);
4332   __ jump_from_interpreted(Rmethod);
4333 }
4334 
4335 //----------------------------------------------------------------------------------------------------
4336 // Allocation
4337 
4338 void TemplateTable::_new() {
4339   transition(vtos, atos);
4340 
4341   const Register Robj   = R0_tos;
4342   const Register Rcpool = R1_tmp;
4343   const Register Rindex = R2_tmp;
4344   const Register Rtags  = R3_tmp;
4345   const Register Rsize  = R3_tmp;
4346 
4347   Register Rklass = R4_tmp;
4348   assert_different_registers(Rcpool, Rindex, Rtags, Rklass, Rtemp);
4349   assert_different_registers(Rcpool, Rindex, Rklass, Rsize);
4350 
4351   Label slow_case;
4352   Label done;
4353   Label initialize_header;
4354   Label initialize_object;  // including clearing the fields
4355   Label allocate_shared;
4356 
4357   const bool allow_shared_alloc =
4358     Universe::heap()->supports_inline_contig_alloc();
4359 
4360   // Literals
4361   InlinedAddress Lheap_top_addr(allow_shared_alloc ? (address)Universe::heap()->top_addr() : NULL);
4362 
4363   __ get_unsigned_2_byte_index_at_bcp(Rindex, 1);
4364   __ get_cpool_and_tags(Rcpool, Rtags);
4365 
4366   // Make sure the class we're about to instantiate has been resolved.
4367   // This is done before loading InstanceKlass to be consistent with the order
4368   // how Constant Pool is updated (see ConstantPool::klass_at_put)
4369   const int tags_offset = Array<u1>::base_offset_in_bytes();
4370   __ add(Rtemp, Rtags, Rindex);
4371 
4372 #ifdef AARCH64
4373   __ add(Rtemp, Rtemp, tags_offset);
4374   __ ldarb(Rtemp, Rtemp);
4375 #else
4376   __ ldrb(Rtemp, Address(Rtemp, tags_offset));
4377 
4378   // use Rklass as a scratch
4379   volatile_barrier(MacroAssembler::LoadLoad, Rklass);
4380 #endif // AARCH64
4381 
4382   // get InstanceKlass
4383   __ cmp(Rtemp, JVM_CONSTANT_Class);
4384   __ b(slow_case, ne);
4385   __ load_resolved_klass_at_offset(Rcpool, Rindex, Rklass);
4386 
4387   // make sure klass is initialized & doesn't have finalizer
4388   // make sure klass is fully initialized
4389   __ ldrb(Rtemp, Address(Rklass, InstanceKlass::init_state_offset()));
4390   __ cmp(Rtemp, InstanceKlass::fully_initialized);
4391   __ b(slow_case, ne);
4392 
4393   // get instance_size in InstanceKlass (scaled to a count of bytes)
4394   __ ldr_u32(Rsize, Address(Rklass, Klass::layout_helper_offset()));
4395 
4396   // test to see if it has a finalizer or is malformed in some way
4397   // Klass::_lh_instance_slow_path_bit is really a bit mask, not bit number
4398   __ tbnz(Rsize, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
4399 
4400   //
4401   // Allocate the instance
4402   // 1) Try to allocate in the TLAB
4403   // 2) if fail and the object is large allocate in the shared Eden
4404   // 3) if the above fails (or is not applicable), go to a slow case
4405   // (creates a new TLAB, etc.)
4406 
4407   if (UseTLAB) {
4408     const Register Rtlab_top = R1_tmp;
4409     const Register Rtlab_end = R2_tmp;
4410     assert_different_registers(Robj, Rsize, Rklass, Rtlab_top, Rtlab_end);
4411 
4412     __ ldr(Robj, Address(Rthread, JavaThread::tlab_top_offset()));
4413     __ ldr(Rtlab_end, Address(Rthread, in_bytes(JavaThread::tlab_end_offset())));
4414     __ add(Rtlab_top, Robj, Rsize);
4415     __ cmp(Rtlab_top, Rtlab_end);
4416     __ b(allow_shared_alloc ? allocate_shared : slow_case, hi);
4417     __ str(Rtlab_top, Address(Rthread, JavaThread::tlab_top_offset()));
4418     if (ZeroTLAB) {
4419       // the fields have been already cleared
4420       __ b(initialize_header);
4421     } else {
4422       // initialize both the header and fields
4423       __ b(initialize_object);
4424     }
4425   }
4426 
4427   // Allocation in the shared Eden, if allowed.
4428   if (allow_shared_alloc) {
4429     __ bind(allocate_shared);
4430 
4431     const Register Rheap_top_addr = R2_tmp;
4432     const Register Rheap_top = R5_tmp;
4433     const Register Rheap_end = Rtemp;
4434     assert_different_registers(Robj, Rklass, Rsize, Rheap_top_addr, Rheap_top, Rheap_end, LR);
4435 
4436     // heap_end now (re)loaded in the loop since also used as a scratch register in the CAS
4437     __ ldr_literal(Rheap_top_addr, Lheap_top_addr);
4438 
4439     Label retry;
4440     __ bind(retry);
4441 
4442 #ifdef AARCH64
4443     __ ldxr(Robj, Rheap_top_addr);
4444 #else
4445     __ ldr(Robj, Address(Rheap_top_addr));
4446 #endif // AARCH64
4447 
4448     __ ldr(Rheap_end, Address(Rheap_top_addr, (intptr_t)Universe::heap()->end_addr()-(intptr_t)Universe::heap()->top_addr()));
4449     __ add(Rheap_top, Robj, Rsize);
4450     __ cmp(Rheap_top, Rheap_end);
4451     __ b(slow_case, hi);
4452 
4453     // Update heap top atomically.
4454     // If someone beats us on the allocation, try again, otherwise continue.
4455 #ifdef AARCH64
4456     __ stxr(Rtemp2, Rheap_top, Rheap_top_addr);
4457     __ cbnz_w(Rtemp2, retry);
4458 #else
4459     __ atomic_cas_bool(Robj, Rheap_top, Rheap_top_addr, 0, Rheap_end/*scratched*/);
4460     __ b(retry, ne);
4461 #endif // AARCH64
4462 
4463     __ incr_allocated_bytes(Rsize, Rtemp);
4464   }
4465 
4466   if (UseTLAB || allow_shared_alloc) {
4467     const Register Rzero0 = R1_tmp;
4468     const Register Rzero1 = R2_tmp;
4469     const Register Rzero_end = R5_tmp;
4470     const Register Rzero_cur = Rtemp;
4471     assert_different_registers(Robj, Rsize, Rklass, Rzero0, Rzero1, Rzero_cur, Rzero_end);
4472 
4473     // The object is initialized before the header.  If the object size is
4474     // zero, go directly to the header initialization.
4475     __ bind(initialize_object);
4476     __ subs(Rsize, Rsize, sizeof(oopDesc));
4477     __ add(Rzero_cur, Robj, sizeof(oopDesc));
4478     __ b(initialize_header, eq);
4479 
4480 #ifdef ASSERT
4481     // make sure Rsize is a multiple of 8
4482     Label L;
4483     __ tst(Rsize, 0x07);
4484     __ b(L, eq);
4485     __ stop("object size is not multiple of 8 - adjust this code");
4486     __ bind(L);
4487 #endif
4488 
4489 #ifdef AARCH64
4490     {
4491       Label loop;
4492       // Step back by 1 word if object size is not a multiple of 2*wordSize.
4493       assert(wordSize <= sizeof(oopDesc), "oop header should contain at least one word");
4494       __ andr(Rtemp2, Rsize, (uintx)wordSize);
4495       __ sub(Rzero_cur, Rzero_cur, Rtemp2);
4496 
4497       // Zero by 2 words per iteration.
4498       __ bind(loop);
4499       __ subs(Rsize, Rsize, 2*wordSize);
4500       __ stp(ZR, ZR, Address(Rzero_cur, 2*wordSize, post_indexed));
4501       __ b(loop, gt);
4502     }
4503 #else
4504     __ mov(Rzero0, 0);
4505     __ mov(Rzero1, 0);
4506     __ add(Rzero_end, Rzero_cur, Rsize);
4507 
4508     // initialize remaining object fields: Rsize was a multiple of 8
4509     { Label loop;
4510       // loop is unrolled 2 times
4511       __ bind(loop);
4512       // #1
4513       __ stmia(Rzero_cur, RegisterSet(Rzero0) | RegisterSet(Rzero1), writeback);
4514       __ cmp(Rzero_cur, Rzero_end);
4515       // #2
4516       __ stmia(Rzero_cur, RegisterSet(Rzero0) | RegisterSet(Rzero1), writeback, ne);
4517       __ cmp(Rzero_cur, Rzero_end, ne);
4518       __ b(loop, ne);
4519     }
4520 #endif // AARCH64
4521 
4522     // initialize object header only.
4523     __ bind(initialize_header);
4524     if (UseBiasedLocking) {
4525       __ ldr(Rtemp, Address(Rklass, Klass::prototype_header_offset()));
4526     } else {
4527       __ mov_slow(Rtemp, (intptr_t)markOopDesc::prototype());
4528     }
4529     // mark
4530     __ str(Rtemp, Address(Robj, oopDesc::mark_offset_in_bytes()));
4531 
4532     // klass
4533 #ifdef AARCH64
4534     __ store_klass_gap(Robj);
4535 #endif // AARCH64
4536     __ store_klass(Rklass, Robj); // blows Rklass:
4537     Rklass = noreg;
4538 
4539     // Note: Disable DTrace runtime check for now to eliminate overhead on each allocation
4540     if (DTraceAllocProbes) {
4541       // Trigger dtrace event for fastpath
4542       Label Lcontinue;
4543 
4544       __ ldrb_global(Rtemp, (address)&DTraceAllocProbes);
4545       __ cbz(Rtemp, Lcontinue);
4546 
4547       __ push(atos);
4548       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), Robj);
4549       __ pop(atos);
4550 
4551       __ bind(Lcontinue);
4552     }
4553 
4554     __ b(done);
4555   } else {
4556     // jump over literals
4557     __ b(slow_case);
4558   }
4559 
4560   if (allow_shared_alloc) {
4561     __ bind_literal(Lheap_top_addr);
4562   }
4563 
4564   // slow case
4565   __ bind(slow_case);
4566   __ get_constant_pool(Rcpool);
4567   __ get_unsigned_2_byte_index_at_bcp(Rindex, 1);
4568   __ call_VM(Robj, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), Rcpool, Rindex);
4569 
4570   // continue
4571   __ bind(done);
4572 
4573   // StoreStore barrier required after complete initialization
4574   // (headers + content zeroing), before the object may escape.
4575   __ membar(MacroAssembler::StoreStore, R1_tmp);
4576 }
4577 
4578 
4579 void TemplateTable::newarray() {
4580   transition(itos, atos);
4581   __ ldrb(R1, at_bcp(1));
4582   __ mov(R2, R0_tos);
4583   call_VM(R0_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray), R1, R2);
4584   // MacroAssembler::StoreStore useless (included in the runtime exit path)
4585 }
4586 
4587 
4588 void TemplateTable::anewarray() {
4589   transition(itos, atos);
4590   __ get_unsigned_2_byte_index_at_bcp(R2, 1);
4591   __ get_constant_pool(R1);
4592   __ mov(R3, R0_tos);
4593   call_VM(R0_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray), R1, R2, R3);
4594   // MacroAssembler::StoreStore useless (included in the runtime exit path)
4595 }
4596 
4597 
4598 void TemplateTable::arraylength() {
4599   transition(atos, itos);
4600   __ null_check(R0_tos, Rtemp, arrayOopDesc::length_offset_in_bytes());
4601   __ ldr_s32(R0_tos, Address(R0_tos, arrayOopDesc::length_offset_in_bytes()));
4602 }
4603 
4604 
4605 void TemplateTable::checkcast() {
4606   transition(atos, atos);
4607   Label done, is_null, quicked, resolved, throw_exception;
4608 
4609   const Register Robj = R0_tos;
4610   const Register Rcpool = R2_tmp;
4611   const Register Rtags = R3_tmp;
4612   const Register Rindex = R4_tmp;
4613   const Register Rsuper = R3_tmp;
4614   const Register Rsub   = R4_tmp;
4615   const Register Rsubtype_check_tmp1 = R1_tmp;
4616   const Register Rsubtype_check_tmp2 = LR_tmp;
4617 
4618   __ cbz(Robj, is_null);
4619 
4620   // Get cpool & tags index
4621   __ get_cpool_and_tags(Rcpool, Rtags);
4622   __ get_unsigned_2_byte_index_at_bcp(Rindex, 1);
4623 
4624   // See if bytecode has already been quicked
4625   __ add(Rtemp, Rtags, Rindex);
4626 #ifdef AARCH64
4627   // TODO-AARCH64: investigate if LoadLoad barrier is needed here or control dependency is enough
4628   __ add(Rtemp, Rtemp, Array<u1>::base_offset_in_bytes());
4629   __ ldarb(Rtemp, Rtemp); // acts as LoadLoad memory barrier
4630 #else
4631   __ ldrb(Rtemp, Address(Rtemp, Array<u1>::base_offset_in_bytes()));
4632 #endif // AARCH64
4633 
4634   __ cmp(Rtemp, JVM_CONSTANT_Class);
4635 
4636 #ifndef AARCH64
4637   volatile_barrier(MacroAssembler::LoadLoad, Rtemp, true);
4638 #endif // !AARCH64
4639 
4640   __ b(quicked, eq);
4641 
4642   __ push(atos);
4643   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4644   // vm_result_2 has metadata result
4645   __ get_vm_result_2(Rsuper, Robj);
4646   __ pop_ptr(Robj);
4647   __ b(resolved);
4648 
4649   __ bind(throw_exception);
4650   // Come here on failure of subtype check
4651   __ profile_typecheck_failed(R1_tmp);
4652   __ mov(R2_ClassCastException_obj, Robj);             // convention with generate_ClassCastException_handler()
4653   __ b(Interpreter::_throw_ClassCastException_entry);
4654 
4655   // Get superklass in Rsuper and subklass in Rsub
4656   __ bind(quicked);
4657   __ load_resolved_klass_at_offset(Rcpool, Rindex, Rsuper);
4658 
4659   __ bind(resolved);
4660   __ load_klass(Rsub, Robj);
4661 
4662   // Generate subtype check. Blows both tmps and Rtemp.
4663   assert_different_registers(Robj, Rsub, Rsuper, Rsubtype_check_tmp1, Rsubtype_check_tmp2, Rtemp);
4664   __ gen_subtype_check(Rsub, Rsuper, throw_exception, Rsubtype_check_tmp1, Rsubtype_check_tmp2);
4665 
4666   // Come here on success
4667 
4668   // Collect counts on whether this check-cast sees NULLs a lot or not.
4669   if (ProfileInterpreter) {
4670     __ b(done);
4671     __ bind(is_null);
4672     __ profile_null_seen(R1_tmp);
4673   } else {
4674     __ bind(is_null);   // same as 'done'
4675   }
4676   __ bind(done);
4677 }
4678 
4679 
4680 void TemplateTable::instanceof() {
4681   // result = 0: obj == NULL or  obj is not an instanceof the specified klass
4682   // result = 1: obj != NULL and obj is     an instanceof the specified klass
4683 
4684   transition(atos, itos);
4685   Label done, is_null, not_subtype, quicked, resolved;
4686 
4687   const Register Robj = R0_tos;
4688   const Register Rcpool = R2_tmp;
4689   const Register Rtags = R3_tmp;
4690   const Register Rindex = R4_tmp;
4691   const Register Rsuper = R3_tmp;
4692   const Register Rsub   = R4_tmp;
4693   const Register Rsubtype_check_tmp1 = R0_tmp;
4694   const Register Rsubtype_check_tmp2 = R1_tmp;
4695 
4696   __ cbz(Robj, is_null);
4697 
4698   __ load_klass(Rsub, Robj);
4699 
4700   // Get cpool & tags index
4701   __ get_cpool_and_tags(Rcpool, Rtags);
4702   __ get_unsigned_2_byte_index_at_bcp(Rindex, 1);
4703 
4704   // See if bytecode has already been quicked
4705   __ add(Rtemp, Rtags, Rindex);
4706 #ifdef AARCH64
4707   // TODO-AARCH64: investigate if LoadLoad barrier is needed here or control dependency is enough
4708   __ add(Rtemp, Rtemp, Array<u1>::base_offset_in_bytes());
4709   __ ldarb(Rtemp, Rtemp); // acts as LoadLoad memory barrier
4710 #else
4711   __ ldrb(Rtemp, Address(Rtemp, Array<u1>::base_offset_in_bytes()));
4712 #endif // AARCH64
4713   __ cmp(Rtemp, JVM_CONSTANT_Class);
4714 
4715 #ifndef AARCH64
4716   volatile_barrier(MacroAssembler::LoadLoad, Rtemp, true);
4717 #endif // !AARCH64
4718 
4719   __ b(quicked, eq);
4720 
4721   __ push(atos);
4722   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4723   // vm_result_2 has metadata result
4724   __ get_vm_result_2(Rsuper, Robj);
4725   __ pop_ptr(Robj);
4726   __ b(resolved);
4727 
4728   // Get superklass in Rsuper and subklass in Rsub
4729   __ bind(quicked);
4730   __ load_resolved_klass_at_offset(Rcpool, Rindex, Rsuper);
4731 
4732   __ bind(resolved);
4733   __ load_klass(Rsub, Robj);
4734 
4735   // Generate subtype check. Blows both tmps and Rtemp.
4736   __ gen_subtype_check(Rsub, Rsuper, not_subtype, Rsubtype_check_tmp1, Rsubtype_check_tmp2);
4737 
4738   // Come here on success
4739   __ mov(R0_tos, 1);
4740   __ b(done);
4741 
4742   __ bind(not_subtype);
4743   // Come here on failure
4744   __ profile_typecheck_failed(R1_tmp);
4745   __ mov(R0_tos, 0);
4746 
4747   // Collect counts on whether this test sees NULLs a lot or not.
4748   if (ProfileInterpreter) {
4749     __ b(done);
4750     __ bind(is_null);
4751     __ profile_null_seen(R1_tmp);
4752   } else {
4753     __ bind(is_null);   // same as 'done'
4754   }
4755   __ bind(done);
4756 }
4757 
4758 
4759 //----------------------------------------------------------------------------------------------------
4760 // Breakpoints
4761 void TemplateTable::_breakpoint() {
4762 
4763   // Note: We get here even if we are single stepping..
4764   // jbug inists on setting breakpoints at every bytecode
4765   // even if we are in single step mode.
4766 
4767   transition(vtos, vtos);
4768 
4769   // get the unpatched byte code
4770   __ mov(R1, Rmethod);
4771   __ mov(R2, Rbcp);
4772   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::get_original_bytecode_at), R1, R2);
4773 #ifdef AARCH64
4774   __ sxtw(Rtmp_save0, R0);
4775 #else
4776   __ mov(Rtmp_save0, R0);
4777 #endif // AARCH64
4778 
4779   // post the breakpoint event
4780   __ mov(R1, Rmethod);
4781   __ mov(R2, Rbcp);
4782   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint), R1, R2);
4783 
4784   // complete the execution of original bytecode
4785   __ mov(R3_bytecode, Rtmp_save0);
4786   __ dispatch_only_normal(vtos);
4787 }
4788 
4789 
4790 //----------------------------------------------------------------------------------------------------
4791 // Exceptions
4792 
4793 void TemplateTable::athrow() {
4794   transition(atos, vtos);
4795   __ mov(Rexception_obj, R0_tos);
4796   __ null_check(Rexception_obj, Rtemp);
4797   __ b(Interpreter::throw_exception_entry());
4798 }
4799 
4800 
4801 //----------------------------------------------------------------------------------------------------
4802 // Synchronization
4803 //
4804 // Note: monitorenter & exit are symmetric routines; which is reflected
4805 //       in the assembly code structure as well
4806 //
4807 // Stack layout:
4808 //
4809 // [expressions  ] <--- Rstack_top        = expression stack top
4810 // ..
4811 // [expressions  ]
4812 // [monitor entry] <--- monitor block top = expression stack bot
4813 // ..
4814 // [monitor entry]
4815 // [frame data   ] <--- monitor block bot
4816 // ...
4817 // [saved FP     ] <--- FP
4818 
4819 
4820 void TemplateTable::monitorenter() {
4821   transition(atos, vtos);
4822 
4823   const Register Robj = R0_tos;
4824   const Register Rentry = R1_tmp;
4825 
4826   // check for NULL object
4827   __ null_check(Robj, Rtemp);
4828 
4829   const int entry_size = (frame::interpreter_frame_monitor_size() * wordSize);
4830   assert (entry_size % StackAlignmentInBytes == 0, "keep stack alignment");
4831   Label allocate_monitor, allocated;
4832 
4833   // initialize entry pointer
4834   __ mov(Rentry, 0);                             // points to free slot or NULL
4835 
4836   // find a free slot in the monitor block (result in Rentry)
4837   { Label loop, exit;
4838     const Register Rcur = R2_tmp;
4839     const Register Rcur_obj = Rtemp;
4840     const Register Rbottom = R3_tmp;
4841     assert_different_registers(Robj, Rentry, Rcur, Rbottom, Rcur_obj);
4842 
4843     __ ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
4844                                  // points to current entry, starting with top-most entry
4845     __ sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize);
4846                                  // points to word before bottom of monitor block
4847 
4848     __ cmp(Rcur, Rbottom);                       // check if there are no monitors
4849 #ifndef AARCH64
4850     __ ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
4851                                                  // prefetch monitor's object for the first iteration
4852 #endif // !AARCH64
4853     __ b(allocate_monitor, eq);                  // there are no monitors, skip searching
4854 
4855     __ bind(loop);
4856 #ifdef AARCH64
4857     __ ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()));
4858 #endif // AARCH64
4859     __ cmp(Rcur_obj, 0);                         // check if current entry is used
4860     __ mov(Rentry, Rcur, eq);                    // if not used then remember entry
4861 
4862     __ cmp(Rcur_obj, Robj);                      // check if current entry is for same object
4863     __ b(exit, eq);                              // if same object then stop searching
4864 
4865     __ add(Rcur, Rcur, entry_size);              // otherwise advance to next entry
4866 
4867     __ cmp(Rcur, Rbottom);                       // check if bottom reached
4868 #ifndef AARCH64
4869     __ ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
4870                                                  // prefetch monitor's object for the next iteration
4871 #endif // !AARCH64
4872     __ b(loop, ne);                              // if not at bottom then check this entry
4873     __ bind(exit);
4874   }
4875 
4876   __ cbnz(Rentry, allocated);                    // check if a slot has been found; if found, continue with that one
4877 
4878   __ bind(allocate_monitor);
4879 
4880   // allocate one if there's no free slot
4881   { Label loop;
4882     assert_different_registers(Robj, Rentry, R2_tmp, Rtemp);
4883 
4884     // 1. compute new pointers
4885 
4886 #ifdef AARCH64
4887     __ check_extended_sp(Rtemp);
4888     __ sub(SP, SP, entry_size);                  // adjust extended SP
4889     __ mov(Rtemp, SP);
4890     __ str(Rtemp, Address(FP, frame::interpreter_frame_extended_sp_offset * wordSize));
4891 #endif // AARCH64
4892 
4893     __ ldr(Rentry, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
4894                                                  // old monitor block top / expression stack bottom
4895 
4896     __ sub(Rstack_top, Rstack_top, entry_size);  // move expression stack top
4897     __ check_stack_top_on_expansion();
4898 
4899     __ sub(Rentry, Rentry, entry_size);          // move expression stack bottom
4900 
4901     __ mov(R2_tmp, Rstack_top);                  // set start value for copy loop
4902 
4903     __ str(Rentry, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
4904                                                  // set new monitor block top
4905 
4906     // 2. move expression stack contents
4907 
4908     __ cmp(R2_tmp, Rentry);                                 // check if expression stack is empty
4909 #ifndef AARCH64
4910     __ ldr(Rtemp, Address(R2_tmp, entry_size), ne);         // load expression stack word from old location
4911 #endif // !AARCH64
4912     __ b(allocated, eq);
4913 
4914     __ bind(loop);
4915 #ifdef AARCH64
4916     __ ldr(Rtemp, Address(R2_tmp, entry_size));             // load expression stack word from old location
4917 #endif // AARCH64
4918     __ str(Rtemp, Address(R2_tmp, wordSize, post_indexed)); // store expression stack word at new location
4919                                                             // and advance to next word
4920     __ cmp(R2_tmp, Rentry);                                 // check if bottom reached
4921 #ifndef AARCH64
4922     __ ldr(Rtemp, Address(R2, entry_size), ne);             // load expression stack word from old location
4923 #endif // !AARCH64
4924     __ b(loop, ne);                                         // if not at bottom then copy next word
4925   }
4926 
4927   // call run-time routine
4928 
4929   // Rentry: points to monitor entry
4930   __ bind(allocated);
4931 
4932   // Increment bcp to point to the next bytecode, so exception handling for async. exceptions work correctly.
4933   // The object has already been poped from the stack, so the expression stack looks correct.
4934   __ add(Rbcp, Rbcp, 1);
4935 
4936   __ str(Robj, Address(Rentry, BasicObjectLock::obj_offset_in_bytes()));     // store object
4937   __ lock_object(Rentry);
4938 
4939   // check to make sure this monitor doesn't cause stack overflow after locking
4940   __ save_bcp();  // in case of exception
4941   __ arm_stack_overflow_check(0, Rtemp);
4942 
4943   // The bcp has already been incremented. Just need to dispatch to next instruction.
4944   __ dispatch_next(vtos);
4945 }
4946 
4947 
4948 void TemplateTable::monitorexit() {
4949   transition(atos, vtos);
4950 
4951   const Register Robj = R0_tos;
4952   const Register Rcur = R1_tmp;
4953   const Register Rbottom = R2_tmp;
4954   const Register Rcur_obj = Rtemp;
4955 
4956   // check for NULL object
4957   __ null_check(Robj, Rtemp);
4958 
4959   const int entry_size = (frame::interpreter_frame_monitor_size() * wordSize);
4960   Label found, throw_exception;
4961 
4962   // find matching slot
4963   { Label loop;
4964     assert_different_registers(Robj, Rcur, Rbottom, Rcur_obj);
4965 
4966     __ ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
4967                                  // points to current entry, starting with top-most entry
4968     __ sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize);
4969                                  // points to word before bottom of monitor block
4970 
4971     __ cmp(Rcur, Rbottom);                       // check if bottom reached
4972 #ifndef AARCH64
4973     __ ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
4974                                                  // prefetch monitor's object for the first iteration
4975 #endif // !AARCH64
4976     __ b(throw_exception, eq);                   // throw exception if there are now monitors
4977 
4978     __ bind(loop);
4979 #ifdef AARCH64
4980     __ ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()));
4981 #endif // AARCH64
4982     // check if current entry is for same object
4983     __ cmp(Rcur_obj, Robj);
4984     __ b(found, eq);                             // if same object then stop searching
4985     __ add(Rcur, Rcur, entry_size);              // otherwise advance to next entry
4986     __ cmp(Rcur, Rbottom);                       // check if bottom reached
4987 #ifndef AARCH64
4988     __ ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
4989 #endif // !AARCH64
4990     __ b (loop, ne);                             // if not at bottom then check this entry
4991   }
4992 
4993   // error handling. Unlocking was not block-structured
4994   __ bind(throw_exception);
4995   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
4996   __ should_not_reach_here();
4997 
4998   // call run-time routine
4999   // Rcur: points to monitor entry
5000   __ bind(found);
5001   __ push_ptr(Robj);                             // make sure object is on stack (contract with oopMaps)
5002   __ unlock_object(Rcur);
5003   __ pop_ptr(Robj);                              // discard object
5004 }
5005 
5006 
5007 //----------------------------------------------------------------------------------------------------
5008 // Wide instructions
5009 
5010 void TemplateTable::wide() {
5011   transition(vtos, vtos);
5012   __ ldrb(R3_bytecode, at_bcp(1));
5013 
5014   InlinedAddress Ltable((address)Interpreter::_wentry_point);
5015   __ ldr_literal(Rtemp, Ltable);
5016   __ indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
5017 
5018   __ nop(); // to avoid filling CPU pipeline with invalid instructions
5019   __ nop();
5020   __ bind_literal(Ltable);
5021 }
5022 
5023 
5024 //----------------------------------------------------------------------------------------------------
5025 // Multi arrays
5026 
5027 void TemplateTable::multianewarray() {
5028   transition(vtos, atos);
5029   __ ldrb(Rtmp_save0, at_bcp(3));   // get number of dimensions
5030 
5031   // last dim is on top of stack; we want address of first one:
5032   // first_addr = last_addr + ndims * stackElementSize - 1*wordsize
5033   // the latter wordSize to point to the beginning of the array.
5034   __ add(Rtemp, Rstack_top, AsmOperand(Rtmp_save0, lsl, Interpreter::logStackElementSize));
5035   __ sub(R1, Rtemp, wordSize);
5036 
5037   call_VM(R0, CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray), R1);
5038   __ add(Rstack_top, Rstack_top, AsmOperand(Rtmp_save0, lsl, Interpreter::logStackElementSize));
5039   // MacroAssembler::StoreStore useless (included in the runtime exit path)
5040 }