1 /*
   2  * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/barrierSetAssembler.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/interp_masm.hpp"
  32 #include "interpreter/templateTable.hpp"
  33 #include "memory/universe.hpp"
  34 #include "oops/methodData.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/objArrayKlass.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/frame.inline.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "runtime/stubRoutines.hpp"
  42 #include "runtime/synchronizer.hpp"
  43 
  44 #define __ _masm->
  45 
  46 // Platform-dependent initialization
  47 
  48 void TemplateTable::pd_initialize() {
  49   // No aarch64 specific initialization
  50 }
  51 
  52 // Address computation: local variables
  53 
  54 static inline Address iaddress(int n) {
  55   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  56 }
  57 
  58 static inline Address laddress(int n) {
  59   return iaddress(n + 1);
  60 }
  61 
  62 static inline Address faddress(int n) {
  63   return iaddress(n);
  64 }
  65 
  66 static inline Address daddress(int n) {
  67   return laddress(n);
  68 }
  69 
  70 static inline Address aaddress(int n) {
  71   return iaddress(n);
  72 }
  73 
  74 static inline Address iaddress(Register r) {
  75   return Address(rlocals, r, Address::lsl(3));
  76 }
  77 
  78 static inline Address laddress(Register r, Register scratch,
  79                                InterpreterMacroAssembler* _masm) {
  80   __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
  81   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  82 }
  83 
  84 static inline Address faddress(Register r) {
  85   return iaddress(r);
  86 }
  87 
  88 static inline Address daddress(Register r, Register scratch,
  89                                InterpreterMacroAssembler* _masm) {
  90   return laddress(r, scratch, _masm);
  91 }
  92 
  93 static inline Address aaddress(Register r) {
  94   return iaddress(r);
  95 }
  96 
  97 static inline Address at_rsp() {
  98   return Address(esp, 0);
  99 }
 100 
 101 // At top of Java expression stack which may be different than esp().  It
 102 // isn't for category 1 objects.
 103 static inline Address at_tos   () {
 104   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 105 }
 106 
 107 static inline Address at_tos_p1() {
 108   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 109 }
 110 
 111 static inline Address at_tos_p2() {
 112   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 113 }
 114 
 115 static inline Address at_tos_p3() {
 116   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 117 }
 118 
 119 static inline Address at_tos_p4() {
 120   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 121 }
 122 
 123 static inline Address at_tos_p5() {
 124   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 125 }
 126 
 127 // Condition conversion
 128 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 129   switch (cc) {
 130   case TemplateTable::equal        : return Assembler::NE;
 131   case TemplateTable::not_equal    : return Assembler::EQ;
 132   case TemplateTable::less         : return Assembler::GE;
 133   case TemplateTable::less_equal   : return Assembler::GT;
 134   case TemplateTable::greater      : return Assembler::LE;
 135   case TemplateTable::greater_equal: return Assembler::LT;
 136   }
 137   ShouldNotReachHere();
 138   return Assembler::EQ;
 139 }
 140 
 141 
 142 // Miscelaneous helper routines
 143 // Store an oop (or NULL) at the Address described by obj.
 144 // If val == noreg this means store a NULL
 145 static void do_oop_store(InterpreterMacroAssembler* _masm,
 146                          Address dst,
 147                          Register val,
 148                          DecoratorSet decorators) {
 149   assert(val == noreg || val == r0, "parameter is just for looks");
 150   __ store_heap_oop(dst, val, r10, r1, noreg, decorators); 
 151 }
 152 
 153 static void do_oop_load(InterpreterMacroAssembler* _masm,
 154                         Address src,
 155                         Register dst,
 156                         DecoratorSet decorators) {
 157   __ load_heap_oop(dst, src, r10, r1, decorators);
 158 }
 159 
 160 Address TemplateTable::at_bcp(int offset) {
 161   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 162   return Address(rbcp, offset);
 163 }
 164 
 165 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 166                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 167                                    int byte_no)
 168 {
 169   if (!RewriteBytecodes)  return;
 170   Label L_patch_done;
 171 
 172   switch (bc) {
 173   case Bytecodes::_fast_qputfield:
 174   case Bytecodes::_fast_aputfield:
 175   case Bytecodes::_fast_bputfield:
 176   case Bytecodes::_fast_zputfield:
 177   case Bytecodes::_fast_cputfield:
 178   case Bytecodes::_fast_dputfield:
 179   case Bytecodes::_fast_fputfield:
 180   case Bytecodes::_fast_iputfield:
 181   case Bytecodes::_fast_lputfield:
 182   case Bytecodes::_fast_sputfield:
 183     {
 184       // We skip bytecode quickening for putfield instructions when
 185       // the put_code written to the constant pool cache is zero.
 186       // This is required so that every execution of this instruction
 187       // calls out to InterpreterRuntime::resolve_get_put to do
 188       // additional, required work.
 189       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 190       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 191       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 192       __ movw(bc_reg, bc);
 193       __ cbzw(temp_reg, L_patch_done);  // don't patch
 194     }
 195     break;
 196   default:
 197     assert(byte_no == -1, "sanity");
 198     // the pair bytecodes have already done the load.
 199     if (load_bc_into_bc_reg) {
 200       __ movw(bc_reg, bc);
 201     }
 202   }
 203 
 204   if (JvmtiExport::can_post_breakpoint()) {
 205     Label L_fast_patch;
 206     // if a breakpoint is present we can't rewrite the stream directly
 207     __ load_unsigned_byte(temp_reg, at_bcp(0));
 208     __ cmpw(temp_reg, Bytecodes::_breakpoint);
 209     __ br(Assembler::NE, L_fast_patch);
 210     // Let breakpoint table handling rewrite to quicker bytecode
 211     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 212     __ b(L_patch_done);
 213     __ bind(L_fast_patch);
 214   }
 215 
 216 #ifdef ASSERT
 217   Label L_okay;
 218   __ load_unsigned_byte(temp_reg, at_bcp(0));
 219   __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
 220   __ br(Assembler::EQ, L_okay);
 221   __ cmpw(temp_reg, bc_reg);
 222   __ br(Assembler::EQ, L_okay);
 223   __ stop("patching the wrong bytecode");
 224   __ bind(L_okay);
 225 #endif
 226 
 227   // patch bytecode
 228   __ strb(bc_reg, at_bcp(0));
 229   __ bind(L_patch_done);
 230 }
 231 
 232 
 233 // Individual instructions
 234 
 235 void TemplateTable::nop() {
 236   transition(vtos, vtos);
 237   // nothing to do
 238 }
 239 
 240 void TemplateTable::shouldnotreachhere() {
 241   transition(vtos, vtos);
 242   __ stop("shouldnotreachhere bytecode");
 243 }
 244 
 245 void TemplateTable::aconst_null()
 246 {
 247   transition(vtos, atos);
 248   __ mov(r0, 0);
 249 }
 250 
 251 void TemplateTable::iconst(int value)
 252 {
 253   transition(vtos, itos);
 254   __ mov(r0, value);
 255 }
 256 
 257 void TemplateTable::lconst(int value)
 258 {
 259   __ mov(r0, value);
 260 }
 261 
 262 void TemplateTable::fconst(int value)
 263 {
 264   transition(vtos, ftos);
 265   switch (value) {
 266   case 0:
 267     __ fmovs(v0, zr);
 268     break;
 269   case 1:
 270     __ fmovs(v0, 1.0);
 271     break;
 272   case 2:
 273     __ fmovs(v0, 2.0);
 274     break;
 275   default:
 276     ShouldNotReachHere();
 277     break;
 278   }
 279 }
 280 
 281 void TemplateTable::dconst(int value)
 282 {
 283   transition(vtos, dtos);
 284   switch (value) {
 285   case 0:
 286     __ fmovd(v0, zr);
 287     break;
 288   case 1:
 289     __ fmovd(v0, 1.0);
 290     break;
 291   case 2:
 292     __ fmovd(v0, 2.0);
 293     break;
 294   default:
 295     ShouldNotReachHere();
 296     break;
 297   }
 298 }
 299 
 300 void TemplateTable::bipush()
 301 {
 302   transition(vtos, itos);
 303   __ load_signed_byte32(r0, at_bcp(1));
 304 }
 305 
 306 void TemplateTable::sipush()
 307 {
 308   transition(vtos, itos);
 309   __ load_unsigned_short(r0, at_bcp(1));
 310   __ revw(r0, r0);
 311   __ asrw(r0, r0, 16);
 312 }
 313 
 314 void TemplateTable::ldc(bool wide)
 315 {
 316   transition(vtos, vtos);
 317   Label call_ldc, notFloat, notClass, notInt, Done;
 318 
 319   if (wide) {
 320     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 321   } else {
 322     __ load_unsigned_byte(r1, at_bcp(1));
 323   }
 324   __ get_cpool_and_tags(r2, r0);
 325 
 326   const int base_offset = ConstantPool::header_size() * wordSize;
 327   const int tags_offset = Array<u1>::base_offset_in_bytes();
 328 
 329   // get type
 330   __ add(r3, r1, tags_offset);
 331   __ lea(r3, Address(r0, r3));
 332   __ ldarb(r3, r3);
 333 
 334   // unresolved class - get the resolved class
 335   __ cmp(r3, (u1)JVM_CONSTANT_UnresolvedClass);
 336   __ br(Assembler::EQ, call_ldc);
 337 
 338   // unresolved class in error state - call into runtime to throw the error
 339   // from the first resolution attempt
 340   __ cmp(r3, (u1)JVM_CONSTANT_UnresolvedClassInError);
 341   __ br(Assembler::EQ, call_ldc);
 342 
 343   // resolved class - need to call vm to get java mirror of the class
 344   __ cmp(r3, (u1)JVM_CONSTANT_Class);
 345   __ br(Assembler::NE, notClass);
 346 
 347   __ bind(call_ldc);
 348   __ mov(c_rarg1, wide);
 349   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 350   __ push_ptr(r0);
 351   __ verify_oop(r0);
 352   __ b(Done);
 353 
 354   __ bind(notClass);
 355   __ cmp(r3, (u1)JVM_CONSTANT_Float);
 356   __ br(Assembler::NE, notFloat);
 357   // ftos
 358   __ adds(r1, r2, r1, Assembler::LSL, 3);
 359   __ ldrs(v0, Address(r1, base_offset));
 360   __ push_f();
 361   __ b(Done);
 362 
 363   __ bind(notFloat);
 364 
 365   __ cmp(r3, (u1)JVM_CONSTANT_Integer);
 366   __ br(Assembler::NE, notInt);
 367 
 368   // itos
 369   __ adds(r1, r2, r1, Assembler::LSL, 3);
 370   __ ldrw(r0, Address(r1, base_offset));
 371   __ push_i(r0);
 372   __ b(Done);
 373 
 374   __ bind(notInt);
 375   condy_helper(Done);
 376 
 377   __ bind(Done);
 378 }
 379 
 380 // Fast path for caching oop constants.
 381 void TemplateTable::fast_aldc(bool wide)
 382 {
 383   transition(vtos, atos);
 384 
 385   Register result = r0;
 386   Register tmp = r1;
 387   Register rarg = r2;
 388 
 389   int index_size = wide ? sizeof(u2) : sizeof(u1);
 390 
 391   Label resolved;
 392 
 393   // We are resolved if the resolved reference cache entry contains a
 394   // non-null object (String, MethodType, etc.)
 395   assert_different_registers(result, tmp);
 396   __ get_cache_index_at_bcp(tmp, 1, index_size);
 397   __ load_resolved_reference_at_index(result, tmp);
 398   __ cbnz(result, resolved);
 399 
 400   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 401 
 402   // first time invocation - must resolve first
 403   __ mov(rarg, (int)bytecode());
 404   __ call_VM(result, entry, rarg);
 405 
 406   __ bind(resolved);
 407 
 408   { // Check for the null sentinel.
 409     // If we just called the VM, it already did the mapping for us,
 410     // but it's harmless to retry.
 411     Label notNull;
 412 
 413     // Stash null_sentinel address to get its value later
 414     __ movptr(rarg, (uintptr_t)Universe::the_null_sentinel_addr());
 415     __ ldr(tmp, Address(rarg));
 416     __ cmpoop(result, tmp);
 417     __ br(Assembler::NE, notNull);
 418     __ mov(result, 0);  // NULL object reference
 419     __ bind(notNull);
 420   }
 421 
 422   if (VerifyOops) {
 423     // Safe to call with 0 result
 424     __ verify_oop(result);
 425   }
 426 }
 427 
 428 void TemplateTable::ldc2_w()
 429 {
 430   transition(vtos, vtos);
 431   Label notDouble, notLong, Done;
 432   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 433 
 434   __ get_cpool_and_tags(r1, r2);
 435   const int base_offset = ConstantPool::header_size() * wordSize;
 436   const int tags_offset = Array<u1>::base_offset_in_bytes();
 437 
 438   // get type
 439   __ lea(r2, Address(r2, r0, Address::lsl(0)));
 440   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 441   __ cmpw(r2, (int)JVM_CONSTANT_Double);
 442   __ br(Assembler::NE, notDouble);
 443 
 444   // dtos
 445   __ lea (r2, Address(r1, r0, Address::lsl(3)));
 446   __ ldrd(v0, Address(r2, base_offset));
 447   __ push_d();
 448   __ b(Done);
 449 
 450   __ bind(notDouble);
 451   __ cmpw(r2, (int)JVM_CONSTANT_Long);
 452   __ br(Assembler::NE, notLong);
 453 
 454   // ltos
 455   __ lea(r0, Address(r1, r0, Address::lsl(3)));
 456   __ ldr(r0, Address(r0, base_offset));
 457   __ push_l();
 458   __ b(Done);
 459 
 460   __ bind(notLong);
 461   condy_helper(Done);
 462 
 463   __ bind(Done);
 464 }
 465 
 466 void TemplateTable::condy_helper(Label& Done)
 467 {
 468   Register obj = r0;
 469   Register rarg = r1;
 470   Register flags = r2;
 471   Register off = r3;
 472 
 473   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 474 
 475   __ mov(rarg, (int) bytecode());
 476   __ call_VM(obj, entry, rarg);
 477 
 478   __ get_vm_result_2(flags, rthread);
 479 
 480   // VMr = obj = base address to find primitive value to push
 481   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 482   __ mov(off, flags);
 483   __ andw(off, off, ConstantPoolCacheEntry::field_index_mask);
 484 
 485   const Address field(obj, off);
 486 
 487   // What sort of thing are we loading?
 488   // x86 uses a shift and mask or wings it with a shift plus assert
 489   // the mask is not needed. aarch64 just uses bitfield extract
 490   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,
 491            ConstantPoolCacheEntry::tos_state_bits);
 492 
 493   switch (bytecode()) {
 494     case Bytecodes::_ldc:
 495     case Bytecodes::_ldc_w:
 496       {
 497         // tos in (itos, ftos, stos, btos, ctos, ztos)
 498         Label notInt, notFloat, notShort, notByte, notChar, notBool;
 499         __ cmpw(flags, itos);
 500         __ br(Assembler::NE, notInt);
 501         // itos
 502         __ ldrw(r0, field);
 503         __ push(itos);
 504         __ b(Done);
 505 
 506         __ bind(notInt);
 507         __ cmpw(flags, ftos);
 508         __ br(Assembler::NE, notFloat);
 509         // ftos
 510         __ load_float(field);
 511         __ push(ftos);
 512         __ b(Done);
 513 
 514         __ bind(notFloat);
 515         __ cmpw(flags, stos);
 516         __ br(Assembler::NE, notShort);
 517         // stos
 518         __ load_signed_short(r0, field);
 519         __ push(stos);
 520         __ b(Done);
 521 
 522         __ bind(notShort);
 523         __ cmpw(flags, btos);
 524         __ br(Assembler::NE, notByte);
 525         // btos
 526         __ load_signed_byte(r0, field);
 527         __ push(btos);
 528         __ b(Done);
 529 
 530         __ bind(notByte);
 531         __ cmpw(flags, ctos);
 532         __ br(Assembler::NE, notChar);
 533         // ctos
 534         __ load_unsigned_short(r0, field);
 535         __ push(ctos);
 536         __ b(Done);
 537 
 538         __ bind(notChar);
 539         __ cmpw(flags, ztos);
 540         __ br(Assembler::NE, notBool);
 541         // ztos
 542         __ load_signed_byte(r0, field);
 543         __ push(ztos);
 544         __ b(Done);
 545 
 546         __ bind(notBool);
 547         break;
 548       }
 549 
 550     case Bytecodes::_ldc2_w:
 551       {
 552         Label notLong, notDouble;
 553         __ cmpw(flags, ltos);
 554         __ br(Assembler::NE, notLong);
 555         // ltos
 556         __ ldr(r0, field);
 557         __ push(ltos);
 558         __ b(Done);
 559 
 560         __ bind(notLong);
 561         __ cmpw(flags, dtos);
 562         __ br(Assembler::NE, notDouble);
 563         // dtos
 564         __ load_double(field);
 565         __ push(dtos);
 566         __ b(Done);
 567 
 568        __ bind(notDouble);
 569         break;
 570       }
 571 
 572     default:
 573       ShouldNotReachHere();
 574     }
 575 
 576     __ stop("bad ldc/condy");
 577 }
 578 
 579 void TemplateTable::locals_index(Register reg, int offset)
 580 {
 581   __ ldrb(reg, at_bcp(offset));
 582   __ neg(reg, reg);
 583 }
 584 
 585 void TemplateTable::iload() {
 586   iload_internal();
 587 }
 588 
 589 void TemplateTable::nofast_iload() {
 590   iload_internal(may_not_rewrite);
 591 }
 592 
 593 void TemplateTable::iload_internal(RewriteControl rc) {
 594   transition(vtos, itos);
 595   if (RewriteFrequentPairs && rc == may_rewrite) {
 596     Label rewrite, done;
 597     Register bc = r4;
 598 
 599     // get next bytecode
 600     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 601 
 602     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 603     // last two iloads in a pair.  Comparing against fast_iload means that
 604     // the next bytecode is neither an iload or a caload, and therefore
 605     // an iload pair.
 606     __ cmpw(r1, Bytecodes::_iload);
 607     __ br(Assembler::EQ, done);
 608 
 609     // if _fast_iload rewrite to _fast_iload2
 610     __ cmpw(r1, Bytecodes::_fast_iload);
 611     __ movw(bc, Bytecodes::_fast_iload2);
 612     __ br(Assembler::EQ, rewrite);
 613 
 614     // if _caload rewrite to _fast_icaload
 615     __ cmpw(r1, Bytecodes::_caload);
 616     __ movw(bc, Bytecodes::_fast_icaload);
 617     __ br(Assembler::EQ, rewrite);
 618 
 619     // else rewrite to _fast_iload
 620     __ movw(bc, Bytecodes::_fast_iload);
 621 
 622     // rewrite
 623     // bc: new bytecode
 624     __ bind(rewrite);
 625     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 626     __ bind(done);
 627 
 628   }
 629 
 630   // do iload, get the local value into tos
 631   locals_index(r1);
 632   __ ldr(r0, iaddress(r1));
 633 
 634 }
 635 
 636 void TemplateTable::fast_iload2()
 637 {
 638   transition(vtos, itos);
 639   locals_index(r1);
 640   __ ldr(r0, iaddress(r1));
 641   __ push(itos);
 642   locals_index(r1, 3);
 643   __ ldr(r0, iaddress(r1));
 644 }
 645 
 646 void TemplateTable::fast_iload()
 647 {
 648   transition(vtos, itos);
 649   locals_index(r1);
 650   __ ldr(r0, iaddress(r1));
 651 }
 652 
 653 void TemplateTable::lload()
 654 {
 655   transition(vtos, ltos);
 656   __ ldrb(r1, at_bcp(1));
 657   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 658   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 659 }
 660 
 661 void TemplateTable::fload()
 662 {
 663   transition(vtos, ftos);
 664   locals_index(r1);
 665   // n.b. we use ldrd here because this is a 64 bit slot
 666   // this is comparable to the iload case
 667   __ ldrd(v0, faddress(r1));
 668 }
 669 
 670 void TemplateTable::dload()
 671 {
 672   transition(vtos, dtos);
 673   __ ldrb(r1, at_bcp(1));
 674   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 675   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 676 }
 677 
 678 void TemplateTable::aload()
 679 {
 680   transition(vtos, atos);
 681   locals_index(r1);
 682   __ ldr(r0, iaddress(r1));
 683 }
 684 
 685 void TemplateTable::locals_index_wide(Register reg) {
 686   __ ldrh(reg, at_bcp(2));
 687   __ rev16w(reg, reg);
 688   __ neg(reg, reg);
 689 }
 690 
 691 void TemplateTable::wide_iload() {
 692   transition(vtos, itos);
 693   locals_index_wide(r1);
 694   __ ldr(r0, iaddress(r1));
 695 }
 696 
 697 void TemplateTable::wide_lload()
 698 {
 699   transition(vtos, ltos);
 700   __ ldrh(r1, at_bcp(2));
 701   __ rev16w(r1, r1);
 702   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 703   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 704 }
 705 
 706 void TemplateTable::wide_fload()
 707 {
 708   transition(vtos, ftos);
 709   locals_index_wide(r1);
 710   // n.b. we use ldrd here because this is a 64 bit slot
 711   // this is comparable to the iload case
 712   __ ldrd(v0, faddress(r1));
 713 }
 714 
 715 void TemplateTable::wide_dload()
 716 {
 717   transition(vtos, dtos);
 718   __ ldrh(r1, at_bcp(2));
 719   __ rev16w(r1, r1);
 720   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 721   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 722 }
 723 
 724 void TemplateTable::wide_aload()
 725 {
 726   transition(vtos, atos);
 727   locals_index_wide(r1);
 728   __ ldr(r0, aaddress(r1));
 729 }
 730 
 731 void TemplateTable::index_check(Register array, Register index)
 732 {
 733   // destroys r1, rscratch1
 734   // check array
 735   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 736   // sign extend index for use by indexed load
 737   // __ movl2ptr(index, index);
 738   // check index
 739   Register length = rscratch1;
 740   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 741   __ cmpw(index, length);
 742   if (index != r1) {
 743     // ??? convention: move aberrant index into r1 for exception message
 744     assert(r1 != array, "different registers");
 745     __ mov(r1, index);
 746   }
 747   Label ok;
 748   __ br(Assembler::LO, ok);
 749   // ??? convention: move array into r3 for exception message
 750    __ mov(r3, array);
 751    __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 752    __ br(rscratch1);
 753   __ bind(ok);
 754 }
 755 
 756 void TemplateTable::iaload()
 757 {
 758   transition(itos, itos);
 759   __ mov(r1, r0);
 760   __ pop_ptr(r0);
 761   // r0: array
 762   // r1: index
 763   index_check(r0, r1); // leaves index in r1, kills rscratch1
 764   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
 765   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(2)), noreg, noreg);
 766 }
 767 
 768 void TemplateTable::laload()
 769 {
 770   transition(itos, ltos);
 771   __ mov(r1, r0);
 772   __ pop_ptr(r0);
 773   // r0: array
 774   // r1: index
 775   index_check(r0, r1); // leaves index in r1, kills rscratch1
 776   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
 777   __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(3)), noreg, noreg);
 778 }
 779 
 780 void TemplateTable::faload()
 781 {
 782   transition(itos, ftos);
 783   __ mov(r1, r0);
 784   __ pop_ptr(r0);
 785   // r0: array
 786   // r1: index
 787   index_check(r0, r1); // leaves index in r1, kills rscratch1
 788   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
 789   __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(2)), noreg, noreg);
 790 }
 791 
 792 void TemplateTable::daload()
 793 {
 794   transition(itos, dtos);
 795   __ mov(r1, r0);
 796   __ pop_ptr(r0);
 797   // r0: array
 798   // r1: index
 799   index_check(r0, r1); // leaves index in r1, kills rscratch1
 800   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
 801   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(3)), noreg, noreg);
 802 }
 803 
 804 void TemplateTable::aaload()
 805 {
 806   transition(itos, atos);
 807   __ mov(r1, r0);
 808   __ pop_ptr(r0);
 809   // r0: array
 810   // r1: index
 811   index_check(r0, r1); // leaves index in r1, kills rscratch1
 812   if (ValueArrayFlatten) {
 813     Label is_flat_array, done;
 814 
 815     __ test_flattened_array_oop(r0, r8 /*temp*/, is_flat_array); 
 816     __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 817     do_oop_load(_masm, Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)), r0, IS_ARRAY);
 818 
 819     __ b(done);
 820     __ bind(is_flat_array);
 821     __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), r0, r1);
 822     __ bind(done);
 823   } else {
 824     __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 825     do_oop_load(_masm, Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)), r0, IS_ARRAY);
 826   }
 827 }
 828 
 829 void TemplateTable::baload()
 830 {
 831   transition(itos, itos);
 832   __ mov(r1, r0);
 833   __ pop_ptr(r0);
 834   // r0: array
 835   // r1: index
 836   index_check(r0, r1); // leaves index in r1, kills rscratch1
 837   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
 838   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(0)), noreg, noreg);
 839 }
 840 
 841 void TemplateTable::caload()
 842 {
 843   transition(itos, itos);
 844   __ mov(r1, r0);
 845   __ pop_ptr(r0);
 846   // r0: array
 847   // r1: index
 848   index_check(r0, r1); // leaves index in r1, kills rscratch1
 849   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 850   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 851 }
 852 
 853 // iload followed by caload frequent pair
 854 void TemplateTable::fast_icaload()
 855 {
 856   transition(vtos, itos);
 857   // load index out of locals
 858   locals_index(r2);
 859   __ ldr(r1, iaddress(r2));
 860 
 861   __ pop_ptr(r0);
 862 
 863   // r0: array
 864   // r1: index
 865   index_check(r0, r1); // leaves index in r1, kills rscratch1
 866   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 867   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 868 }
 869 
 870 void TemplateTable::saload()
 871 {
 872   transition(itos, itos);
 873   __ mov(r1, r0);
 874   __ pop_ptr(r0);
 875   // r0: array
 876   // r1: index
 877   index_check(r0, r1); // leaves index in r1, kills rscratch1
 878   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_SHORT) >> 1);
 879   __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 880 }
 881 
 882 void TemplateTable::iload(int n)
 883 {
 884   transition(vtos, itos);
 885   __ ldr(r0, iaddress(n));
 886 }
 887 
 888 void TemplateTable::lload(int n)
 889 {
 890   transition(vtos, ltos);
 891   __ ldr(r0, laddress(n));
 892 }
 893 
 894 void TemplateTable::fload(int n)
 895 {
 896   transition(vtos, ftos);
 897   __ ldrs(v0, faddress(n));
 898 }
 899 
 900 void TemplateTable::dload(int n)
 901 {
 902   transition(vtos, dtos);
 903   __ ldrd(v0, daddress(n));
 904 }
 905 
 906 void TemplateTable::aload(int n)
 907 {
 908   transition(vtos, atos);
 909   __ ldr(r0, iaddress(n));
 910 }
 911 
 912 void TemplateTable::aload_0() {
 913   aload_0_internal();
 914 }
 915 
 916 void TemplateTable::nofast_aload_0() {
 917   aload_0_internal(may_not_rewrite);
 918 }
 919 
 920 void TemplateTable::aload_0_internal(RewriteControl rc) {
 921   // According to bytecode histograms, the pairs:
 922   //
 923   // _aload_0, _fast_igetfield
 924   // _aload_0, _fast_agetfield
 925   // _aload_0, _fast_fgetfield
 926   //
 927   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 928   // _aload_0 bytecode checks if the next bytecode is either
 929   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 930   // rewrites the current bytecode into a pair bytecode; otherwise it
 931   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 932   // the pair check anymore.
 933   //
 934   // Note: If the next bytecode is _getfield, the rewrite must be
 935   //       delayed, otherwise we may miss an opportunity for a pair.
 936   //
 937   // Also rewrite frequent pairs
 938   //   aload_0, aload_1
 939   //   aload_0, iload_1
 940   // These bytecodes with a small amount of code are most profitable
 941   // to rewrite
 942   if (RewriteFrequentPairs && rc == may_rewrite) {
 943     Label rewrite, done;
 944     const Register bc = r4;
 945 
 946     // get next bytecode
 947     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 948 
 949     // if _getfield then wait with rewrite
 950     __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
 951     __ br(Assembler::EQ, done);
 952 
 953     // if _igetfield then rewrite to _fast_iaccess_0
 954     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 955     __ cmpw(r1, Bytecodes::_fast_igetfield);
 956     __ movw(bc, Bytecodes::_fast_iaccess_0);
 957     __ br(Assembler::EQ, rewrite);
 958 
 959     // if _agetfield then rewrite to _fast_aaccess_0
 960     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 961     __ cmpw(r1, Bytecodes::_fast_agetfield);
 962     __ movw(bc, Bytecodes::_fast_aaccess_0);
 963     __ br(Assembler::EQ, rewrite);
 964 
 965     // if _fgetfield then rewrite to _fast_faccess_0
 966     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 967     __ cmpw(r1, Bytecodes::_fast_fgetfield);
 968     __ movw(bc, Bytecodes::_fast_faccess_0);
 969     __ br(Assembler::EQ, rewrite);
 970 
 971     // else rewrite to _fast_aload0
 972     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 973     __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
 974 
 975     // rewrite
 976     // bc: new bytecode
 977     __ bind(rewrite);
 978     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 979 
 980     __ bind(done);
 981   }
 982 
 983   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 984   aload(0);
 985 }
 986 
 987 void TemplateTable::istore()
 988 {
 989   transition(itos, vtos);
 990   locals_index(r1);
 991   // FIXME: We're being very pernickerty here storing a jint in a
 992   // local with strw, which costs an extra instruction over what we'd
 993   // be able to do with a simple str.  We should just store the whole
 994   // word.
 995   __ lea(rscratch1, iaddress(r1));
 996   __ strw(r0, Address(rscratch1));
 997 }
 998 
 999 void TemplateTable::lstore()
1000 {
1001   transition(ltos, vtos);
1002   locals_index(r1);
1003   __ str(r0, laddress(r1, rscratch1, _masm));
1004 }
1005 
1006 void TemplateTable::fstore() {
1007   transition(ftos, vtos);
1008   locals_index(r1);
1009   __ lea(rscratch1, iaddress(r1));
1010   __ strs(v0, Address(rscratch1));
1011 }
1012 
1013 void TemplateTable::dstore() {
1014   transition(dtos, vtos);
1015   locals_index(r1);
1016   __ strd(v0, daddress(r1, rscratch1, _masm));
1017 }
1018 
1019 void TemplateTable::astore()
1020 {
1021   transition(vtos, vtos);
1022   __ pop_ptr(r0);
1023   locals_index(r1);
1024   __ str(r0, aaddress(r1));
1025 }
1026 
1027 void TemplateTable::wide_istore() {
1028   transition(vtos, vtos);
1029   __ pop_i();
1030   locals_index_wide(r1);
1031   __ lea(rscratch1, iaddress(r1));
1032   __ strw(r0, Address(rscratch1));
1033 }
1034 
1035 void TemplateTable::wide_lstore() {
1036   transition(vtos, vtos);
1037   __ pop_l();
1038   locals_index_wide(r1);
1039   __ str(r0, laddress(r1, rscratch1, _masm));
1040 }
1041 
1042 void TemplateTable::wide_fstore() {
1043   transition(vtos, vtos);
1044   __ pop_f();
1045   locals_index_wide(r1);
1046   __ lea(rscratch1, faddress(r1));
1047   __ strs(v0, rscratch1);
1048 }
1049 
1050 void TemplateTable::wide_dstore() {
1051   transition(vtos, vtos);
1052   __ pop_d();
1053   locals_index_wide(r1);
1054   __ strd(v0, daddress(r1, rscratch1, _masm));
1055 }
1056 
1057 void TemplateTable::wide_astore() {
1058   transition(vtos, vtos);
1059   __ pop_ptr(r0);
1060   locals_index_wide(r1);
1061   __ str(r0, aaddress(r1));
1062 }
1063 
1064 void TemplateTable::iastore() {
1065   transition(itos, vtos);
1066   __ pop_i(r1);
1067   __ pop_ptr(r3);
1068   // r0: value
1069   // r1: index
1070   // r3: array
1071   index_check(r3, r1); // prefer index in r1
1072   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
1073   __ access_store_at(T_INT, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(2)), r0, noreg, noreg);
1074 }
1075 
1076 void TemplateTable::lastore() {
1077   transition(ltos, vtos);
1078   __ pop_i(r1);
1079   __ pop_ptr(r3);
1080   // r0: value
1081   // r1: index
1082   // r3: array
1083   index_check(r3, r1); // prefer index in r1
1084   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
1085   __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(3)), r0, noreg, noreg);
1086 }
1087 
1088 void TemplateTable::fastore() {
1089   transition(ftos, vtos);
1090   __ pop_i(r1);
1091   __ pop_ptr(r3);
1092   // v0: value
1093   // r1:  index
1094   // r3:  array
1095   index_check(r3, r1); // prefer index in r1
1096   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
1097   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(2)), noreg /* ftos */, noreg, noreg);
1098 }
1099 
1100 void TemplateTable::dastore() {
1101   transition(dtos, vtos);
1102   __ pop_i(r1);
1103   __ pop_ptr(r3);
1104   // v0: value
1105   // r1:  index
1106   // r3:  array
1107   index_check(r3, r1); // prefer index in r1
1108   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
1109   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(3)), noreg /* dtos */, noreg, noreg);
1110 }
1111 
1112 void TemplateTable::aastore() {
1113   Label is_null, ok_is_subtype, done;
1114   transition(vtos, vtos);
1115   // stack: ..., array, index, value
1116   __ ldr(r0, at_tos());    // value 
1117   __ ldr(r2, at_tos_p1()); // index
1118   __ ldr(r3, at_tos_p2()); // array
1119 
1120   Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop));
1121 
1122   index_check(r3, r2);     // kills r1
1123 
1124   // DMS CHECK: what does line below do?
1125   __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); 
1126 
1127   // do array store check - check for NULL value first
1128   __ cbz(r0, is_null);
1129 
1130   Label  is_flat_array;
1131   if (ValueArrayFlatten) {
1132     __ test_flattened_array_oop(r3, r8 /*temp*/, is_flat_array);
1133   }
1134 
1135   // Move subklass into r1
1136   __ load_klass(r1, r0);
1137 
1138   // Move superklass into r0
1139   __ load_klass(r0, r3);
1140   __ ldr(r0, Address(r0, ObjArrayKlass::element_klass_offset()));
1141   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1142 
1143   // Generate subtype check.  Blows r2, r5
1144   // Superklass in r0.  Subklass in r1.
1145 
1146   __ gen_subtype_check(r1, ok_is_subtype);
1147 
1148   // Come here on failure
1149   // object is at TOS
1150   __ b(Interpreter::_throw_ArrayStoreException_entry);
1151 
1152 
1153   // Come here on success
1154   __ bind(ok_is_subtype);
1155 
1156 
1157   // Get the value we will store
1158   __ ldr(r0, at_tos());
1159   // Now store using the appropriate barrier
1160   do_oop_store(_masm, element_address, r0, IS_ARRAY);
1161   __ b(done);
1162 
1163   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1164   __ bind(is_null);
1165   __ profile_null_seen(r2);
1166 
1167   if (EnableValhalla) {
1168     Label is_null_into_value_array_npe, store_null;
1169 
1170     // No way to store null in flat array
1171     __ test_null_free_array_oop(r3, r8, is_null_into_value_array_npe); 
1172     __ b(store_null);
1173 
1174     __ bind(is_null_into_value_array_npe);
1175     __ b(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
1176 
1177     __ bind(store_null);
1178   }
1179 
1180   // Store a NULL
1181   do_oop_store(_masm, element_address, noreg, IS_ARRAY);
1182   __ b(done);
1183 
1184   if (EnableValhalla) { 
1185      Label is_type_ok;
1186 
1187     // store non-null value
1188     __ bind(is_flat_array);
1189 
1190     // Simplistic type check...
1191     // r0 - value, r2 - index, r3 - array.
1192 
1193     // Profile the not-null value's klass.
1194     // Load value class 
1195      __ load_klass(r1, r0);
1196      __ profile_typecheck(r2, r1, r0); // blows r2, and r0
1197 
1198     // flat value array needs exact type match
1199     // is "r8 == r0" (value subclass == array element superclass)
1200 
1201     // Move element klass into r0
1202 
1203      __ load_klass(r0, r3);
1204 
1205      __ ldr(r0, Address(r0, ArrayKlass::element_klass_offset())); 
1206      __ cmp(r0, r1);
1207      __ br(Assembler::EQ, is_type_ok);
1208 
1209      __ profile_typecheck_failed(r2);
1210      __ b(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
1211 
1212      __ bind(is_type_ok);
1213 
1214     // DMS CHECK: Reload from TOS to be safe, because of profile_typecheck that blows r2 and r0.
1215     // Should we really do it?
1216      __ ldr(r1, at_tos());  // value
1217      __ mov(r2, r3); // array, ldr(r2, at_tos_p2()); 
1218      __ ldr(r3, at_tos_p1()); // index
1219      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_store), r1, r2, r3);
1220   }
1221 
1222 
1223   // Pop stack arguments
1224   __ bind(done);
1225   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1226 }
1227 
1228 void TemplateTable::bastore()
1229 {
1230   transition(itos, vtos);
1231   __ pop_i(r1);
1232   __ pop_ptr(r3);
1233   // r0: value
1234   // r1: index
1235   // r3: array
1236   index_check(r3, r1); // prefer index in r1
1237 
1238   // Need to check whether array is boolean or byte
1239   // since both types share the bastore bytecode.
1240   __ load_klass(r2, r3);
1241   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1242   int diffbit_index = exact_log2(Klass::layout_helper_boolean_diffbit());
1243   Label L_skip;
1244   __ tbz(r2, diffbit_index, L_skip);
1245   __ andw(r0, r0, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1246   __ bind(L_skip);
1247 
1248   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
1249   __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(0)), r0, noreg, noreg);
1250 }
1251 
1252 void TemplateTable::castore()
1253 {
1254   transition(itos, vtos);
1255   __ pop_i(r1);
1256   __ pop_ptr(r3);
1257   // r0: value
1258   // r1: index
1259   // r3: array
1260   index_check(r3, r1); // prefer index in r1
1261   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
1262   __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(1)), r0, noreg, noreg);
1263 }
1264 
1265 void TemplateTable::sastore()
1266 {
1267   castore();
1268 }
1269 
1270 void TemplateTable::istore(int n)
1271 {
1272   transition(itos, vtos);
1273   __ str(r0, iaddress(n));
1274 }
1275 
1276 void TemplateTable::lstore(int n)
1277 {
1278   transition(ltos, vtos);
1279   __ str(r0, laddress(n));
1280 }
1281 
1282 void TemplateTable::fstore(int n)
1283 {
1284   transition(ftos, vtos);
1285   __ strs(v0, faddress(n));
1286 }
1287 
1288 void TemplateTable::dstore(int n)
1289 {
1290   transition(dtos, vtos);
1291   __ strd(v0, daddress(n));
1292 }
1293 
1294 void TemplateTable::astore(int n)
1295 {
1296   transition(vtos, vtos);
1297   __ pop_ptr(r0);
1298   __ str(r0, iaddress(n));
1299 }
1300 
1301 void TemplateTable::pop()
1302 {
1303   transition(vtos, vtos);
1304   __ add(esp, esp, Interpreter::stackElementSize);
1305 }
1306 
1307 void TemplateTable::pop2()
1308 {
1309   transition(vtos, vtos);
1310   __ add(esp, esp, 2 * Interpreter::stackElementSize);
1311 }
1312 
1313 void TemplateTable::dup()
1314 {
1315   transition(vtos, vtos);
1316   __ ldr(r0, Address(esp, 0));
1317   __ push(r0);
1318   // stack: ..., a, a
1319 }
1320 
1321 void TemplateTable::dup_x1()
1322 {
1323   transition(vtos, vtos);
1324   // stack: ..., a, b
1325   __ ldr(r0, at_tos());  // load b
1326   __ ldr(r2, at_tos_p1());  // load a
1327   __ str(r0, at_tos_p1());  // store b
1328   __ str(r2, at_tos());  // store a
1329   __ push(r0);                  // push b
1330   // stack: ..., b, a, b
1331 }
1332 
1333 void TemplateTable::dup_x2()
1334 {
1335   transition(vtos, vtos);
1336   // stack: ..., a, b, c
1337   __ ldr(r0, at_tos());  // load c
1338   __ ldr(r2, at_tos_p2());  // load a
1339   __ str(r0, at_tos_p2());  // store c in a
1340   __ push(r0);      // push c
1341   // stack: ..., c, b, c, c
1342   __ ldr(r0, at_tos_p2());  // load b
1343   __ str(r2, at_tos_p2());  // store a in b
1344   // stack: ..., c, a, c, c
1345   __ str(r0, at_tos_p1());  // store b in c
1346   // stack: ..., c, a, b, c
1347 }
1348 
1349 void TemplateTable::dup2()
1350 {
1351   transition(vtos, vtos);
1352   // stack: ..., a, b
1353   __ ldr(r0, at_tos_p1());  // load a
1354   __ push(r0);                  // push a
1355   __ ldr(r0, at_tos_p1());  // load b
1356   __ push(r0);                  // push b
1357   // stack: ..., a, b, a, b
1358 }
1359 
1360 void TemplateTable::dup2_x1()
1361 {
1362   transition(vtos, vtos);
1363   // stack: ..., a, b, c
1364   __ ldr(r2, at_tos());  // load c
1365   __ ldr(r0, at_tos_p1());  // load b
1366   __ push(r0);                  // push b
1367   __ push(r2);                  // push c
1368   // stack: ..., a, b, c, b, c
1369   __ str(r2, at_tos_p3());  // store c in b
1370   // stack: ..., a, c, c, b, c
1371   __ ldr(r2, at_tos_p4());  // load a
1372   __ str(r2, at_tos_p2());  // store a in 2nd c
1373   // stack: ..., a, c, a, b, c
1374   __ str(r0, at_tos_p4());  // store b in a
1375   // stack: ..., b, c, a, b, c
1376 }
1377 
1378 void TemplateTable::dup2_x2()
1379 {
1380   transition(vtos, vtos);
1381   // stack: ..., a, b, c, d
1382   __ ldr(r2, at_tos());  // load d
1383   __ ldr(r0, at_tos_p1());  // load c
1384   __ push(r0)            ;      // push c
1385   __ push(r2);                  // push d
1386   // stack: ..., a, b, c, d, c, d
1387   __ ldr(r0, at_tos_p4());  // load b
1388   __ str(r0, at_tos_p2());  // store b in d
1389   __ str(r2, at_tos_p4());  // store d in b
1390   // stack: ..., a, d, c, b, c, d
1391   __ ldr(r2, at_tos_p5());  // load a
1392   __ ldr(r0, at_tos_p3());  // load c
1393   __ str(r2, at_tos_p3());  // store a in c
1394   __ str(r0, at_tos_p5());  // store c in a
1395   // stack: ..., c, d, a, b, c, d
1396 }
1397 
1398 void TemplateTable::swap()
1399 {
1400   transition(vtos, vtos);
1401   // stack: ..., a, b
1402   __ ldr(r2, at_tos_p1());  // load a
1403   __ ldr(r0, at_tos());  // load b
1404   __ str(r2, at_tos());  // store a in b
1405   __ str(r0, at_tos_p1());  // store b in a
1406   // stack: ..., b, a
1407 }
1408 
1409 void TemplateTable::iop2(Operation op)
1410 {
1411   transition(itos, itos);
1412   // r0 <== r1 op r0
1413   __ pop_i(r1);
1414   switch (op) {
1415   case add  : __ addw(r0, r1, r0); break;
1416   case sub  : __ subw(r0, r1, r0); break;
1417   case mul  : __ mulw(r0, r1, r0); break;
1418   case _and : __ andw(r0, r1, r0); break;
1419   case _or  : __ orrw(r0, r1, r0); break;
1420   case _xor : __ eorw(r0, r1, r0); break;
1421   case shl  : __ lslvw(r0, r1, r0); break;
1422   case shr  : __ asrvw(r0, r1, r0); break;
1423   case ushr : __ lsrvw(r0, r1, r0);break;
1424   default   : ShouldNotReachHere();
1425   }
1426 }
1427 
1428 void TemplateTable::lop2(Operation op)
1429 {
1430   transition(ltos, ltos);
1431   // r0 <== r1 op r0
1432   __ pop_l(r1);
1433   switch (op) {
1434   case add  : __ add(r0, r1, r0); break;
1435   case sub  : __ sub(r0, r1, r0); break;
1436   case mul  : __ mul(r0, r1, r0); break;
1437   case _and : __ andr(r0, r1, r0); break;
1438   case _or  : __ orr(r0, r1, r0); break;
1439   case _xor : __ eor(r0, r1, r0); break;
1440   default   : ShouldNotReachHere();
1441   }
1442 }
1443 
1444 void TemplateTable::idiv()
1445 {
1446   transition(itos, itos);
1447   // explicitly check for div0
1448   Label no_div0;
1449   __ cbnzw(r0, no_div0);
1450   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1451   __ br(rscratch1);
1452   __ bind(no_div0);
1453   __ pop_i(r1);
1454   // r0 <== r1 idiv r0
1455   __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1456 }
1457 
1458 void TemplateTable::irem()
1459 {
1460   transition(itos, itos);
1461   // explicitly check for div0
1462   Label no_div0;
1463   __ cbnzw(r0, no_div0);
1464   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1465   __ br(rscratch1);
1466   __ bind(no_div0);
1467   __ pop_i(r1);
1468   // r0 <== r1 irem r0
1469   __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1470 }
1471 
1472 void TemplateTable::lmul()
1473 {
1474   transition(ltos, ltos);
1475   __ pop_l(r1);
1476   __ mul(r0, r0, r1);
1477 }
1478 
1479 void TemplateTable::ldiv()
1480 {
1481   transition(ltos, ltos);
1482   // explicitly check for div0
1483   Label no_div0;
1484   __ cbnz(r0, no_div0);
1485   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1486   __ br(rscratch1);
1487   __ bind(no_div0);
1488   __ pop_l(r1);
1489   // r0 <== r1 ldiv r0
1490   __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1491 }
1492 
1493 void TemplateTable::lrem()
1494 {
1495   transition(ltos, ltos);
1496   // explicitly check for div0
1497   Label no_div0;
1498   __ cbnz(r0, no_div0);
1499   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1500   __ br(rscratch1);
1501   __ bind(no_div0);
1502   __ pop_l(r1);
1503   // r0 <== r1 lrem r0
1504   __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1505 }
1506 
1507 void TemplateTable::lshl()
1508 {
1509   transition(itos, ltos);
1510   // shift count is in r0
1511   __ pop_l(r1);
1512   __ lslv(r0, r1, r0);
1513 }
1514 
1515 void TemplateTable::lshr()
1516 {
1517   transition(itos, ltos);
1518   // shift count is in r0
1519   __ pop_l(r1);
1520   __ asrv(r0, r1, r0);
1521 }
1522 
1523 void TemplateTable::lushr()
1524 {
1525   transition(itos, ltos);
1526   // shift count is in r0
1527   __ pop_l(r1);
1528   __ lsrv(r0, r1, r0);
1529 }
1530 
1531 void TemplateTable::fop2(Operation op)
1532 {
1533   transition(ftos, ftos);
1534   switch (op) {
1535   case add:
1536     // n.b. use ldrd because this is a 64 bit slot
1537     __ pop_f(v1);
1538     __ fadds(v0, v1, v0);
1539     break;
1540   case sub:
1541     __ pop_f(v1);
1542     __ fsubs(v0, v1, v0);
1543     break;
1544   case mul:
1545     __ pop_f(v1);
1546     __ fmuls(v0, v1, v0);
1547     break;
1548   case div:
1549     __ pop_f(v1);
1550     __ fdivs(v0, v1, v0);
1551     break;
1552   case rem:
1553     __ fmovs(v1, v0);
1554     __ pop_f(v0);
1555     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::frem),
1556                          0, 2, MacroAssembler::ret_type_float);
1557     break;
1558   default:
1559     ShouldNotReachHere();
1560     break;
1561   }
1562 }
1563 
1564 void TemplateTable::dop2(Operation op)
1565 {
1566   transition(dtos, dtos);
1567   switch (op) {
1568   case add:
1569     // n.b. use ldrd because this is a 64 bit slot
1570     __ pop_d(v1);
1571     __ faddd(v0, v1, v0);
1572     break;
1573   case sub:
1574     __ pop_d(v1);
1575     __ fsubd(v0, v1, v0);
1576     break;
1577   case mul:
1578     __ pop_d(v1);
1579     __ fmuld(v0, v1, v0);
1580     break;
1581   case div:
1582     __ pop_d(v1);
1583     __ fdivd(v0, v1, v0);
1584     break;
1585   case rem:
1586     __ fmovd(v1, v0);
1587     __ pop_d(v0);
1588     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::drem),
1589                          0, 2, MacroAssembler::ret_type_double);
1590     break;
1591   default:
1592     ShouldNotReachHere();
1593     break;
1594   }
1595 }
1596 
1597 void TemplateTable::ineg()
1598 {
1599   transition(itos, itos);
1600   __ negw(r0, r0);
1601 
1602 }
1603 
1604 void TemplateTable::lneg()
1605 {
1606   transition(ltos, ltos);
1607   __ neg(r0, r0);
1608 }
1609 
1610 void TemplateTable::fneg()
1611 {
1612   transition(ftos, ftos);
1613   __ fnegs(v0, v0);
1614 }
1615 
1616 void TemplateTable::dneg()
1617 {
1618   transition(dtos, dtos);
1619   __ fnegd(v0, v0);
1620 }
1621 
1622 void TemplateTable::iinc()
1623 {
1624   transition(vtos, vtos);
1625   __ load_signed_byte(r1, at_bcp(2)); // get constant
1626   locals_index(r2);
1627   __ ldr(r0, iaddress(r2));
1628   __ addw(r0, r0, r1);
1629   __ str(r0, iaddress(r2));
1630 }
1631 
1632 void TemplateTable::wide_iinc()
1633 {
1634   transition(vtos, vtos);
1635   // __ mov(r1, zr);
1636   __ ldrw(r1, at_bcp(2)); // get constant and index
1637   __ rev16(r1, r1);
1638   __ ubfx(r2, r1, 0, 16);
1639   __ neg(r2, r2);
1640   __ sbfx(r1, r1, 16, 16);
1641   __ ldr(r0, iaddress(r2));
1642   __ addw(r0, r0, r1);
1643   __ str(r0, iaddress(r2));
1644 }
1645 
1646 void TemplateTable::convert()
1647 {
1648   // Checking
1649 #ifdef ASSERT
1650   {
1651     TosState tos_in  = ilgl;
1652     TosState tos_out = ilgl;
1653     switch (bytecode()) {
1654     case Bytecodes::_i2l: // fall through
1655     case Bytecodes::_i2f: // fall through
1656     case Bytecodes::_i2d: // fall through
1657     case Bytecodes::_i2b: // fall through
1658     case Bytecodes::_i2c: // fall through
1659     case Bytecodes::_i2s: tos_in = itos; break;
1660     case Bytecodes::_l2i: // fall through
1661     case Bytecodes::_l2f: // fall through
1662     case Bytecodes::_l2d: tos_in = ltos; break;
1663     case Bytecodes::_f2i: // fall through
1664     case Bytecodes::_f2l: // fall through
1665     case Bytecodes::_f2d: tos_in = ftos; break;
1666     case Bytecodes::_d2i: // fall through
1667     case Bytecodes::_d2l: // fall through
1668     case Bytecodes::_d2f: tos_in = dtos; break;
1669     default             : ShouldNotReachHere();
1670     }
1671     switch (bytecode()) {
1672     case Bytecodes::_l2i: // fall through
1673     case Bytecodes::_f2i: // fall through
1674     case Bytecodes::_d2i: // fall through
1675     case Bytecodes::_i2b: // fall through
1676     case Bytecodes::_i2c: // fall through
1677     case Bytecodes::_i2s: tos_out = itos; break;
1678     case Bytecodes::_i2l: // fall through
1679     case Bytecodes::_f2l: // fall through
1680     case Bytecodes::_d2l: tos_out = ltos; break;
1681     case Bytecodes::_i2f: // fall through
1682     case Bytecodes::_l2f: // fall through
1683     case Bytecodes::_d2f: tos_out = ftos; break;
1684     case Bytecodes::_i2d: // fall through
1685     case Bytecodes::_l2d: // fall through
1686     case Bytecodes::_f2d: tos_out = dtos; break;
1687     default             : ShouldNotReachHere();
1688     }
1689     transition(tos_in, tos_out);
1690   }
1691 #endif // ASSERT
1692   // static const int64_t is_nan = 0x8000000000000000L;
1693 
1694   // Conversion
1695   switch (bytecode()) {
1696   case Bytecodes::_i2l:
1697     __ sxtw(r0, r0);
1698     break;
1699   case Bytecodes::_i2f:
1700     __ scvtfws(v0, r0);
1701     break;
1702   case Bytecodes::_i2d:
1703     __ scvtfwd(v0, r0);
1704     break;
1705   case Bytecodes::_i2b:
1706     __ sxtbw(r0, r0);
1707     break;
1708   case Bytecodes::_i2c:
1709     __ uxthw(r0, r0);
1710     break;
1711   case Bytecodes::_i2s:
1712     __ sxthw(r0, r0);
1713     break;
1714   case Bytecodes::_l2i:
1715     __ uxtw(r0, r0);
1716     break;
1717   case Bytecodes::_l2f:
1718     __ scvtfs(v0, r0);
1719     break;
1720   case Bytecodes::_l2d:
1721     __ scvtfd(v0, r0);
1722     break;
1723   case Bytecodes::_f2i:
1724   {
1725     Label L_Okay;
1726     __ clear_fpsr();
1727     __ fcvtzsw(r0, v0);
1728     __ get_fpsr(r1);
1729     __ cbzw(r1, L_Okay);
1730     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2i),
1731                          0, 1, MacroAssembler::ret_type_integral);
1732     __ bind(L_Okay);
1733   }
1734     break;
1735   case Bytecodes::_f2l:
1736   {
1737     Label L_Okay;
1738     __ clear_fpsr();
1739     __ fcvtzs(r0, v0);
1740     __ get_fpsr(r1);
1741     __ cbzw(r1, L_Okay);
1742     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2l),
1743                          0, 1, MacroAssembler::ret_type_integral);
1744     __ bind(L_Okay);
1745   }
1746     break;
1747   case Bytecodes::_f2d:
1748     __ fcvts(v0, v0);
1749     break;
1750   case Bytecodes::_d2i:
1751   {
1752     Label L_Okay;
1753     __ clear_fpsr();
1754     __ fcvtzdw(r0, v0);
1755     __ get_fpsr(r1);
1756     __ cbzw(r1, L_Okay);
1757     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2i),
1758                          0, 1, MacroAssembler::ret_type_integral);
1759     __ bind(L_Okay);
1760   }
1761     break;
1762   case Bytecodes::_d2l:
1763   {
1764     Label L_Okay;
1765     __ clear_fpsr();
1766     __ fcvtzd(r0, v0);
1767     __ get_fpsr(r1);
1768     __ cbzw(r1, L_Okay);
1769     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2l),
1770                          0, 1, MacroAssembler::ret_type_integral);
1771     __ bind(L_Okay);
1772   }
1773     break;
1774   case Bytecodes::_d2f:
1775     __ fcvtd(v0, v0);
1776     break;
1777   default:
1778     ShouldNotReachHere();
1779   }
1780 }
1781 
1782 void TemplateTable::lcmp()
1783 {
1784   transition(ltos, itos);
1785   Label done;
1786   __ pop_l(r1);
1787   __ cmp(r1, r0);
1788   __ mov(r0, (u_int64_t)-1L);
1789   __ br(Assembler::LT, done);
1790   // __ mov(r0, 1UL);
1791   // __ csel(r0, r0, zr, Assembler::NE);
1792   // and here is a faster way
1793   __ csinc(r0, zr, zr, Assembler::EQ);
1794   __ bind(done);
1795 }
1796 
1797 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1798 {
1799   Label done;
1800   if (is_float) {
1801     // XXX get rid of pop here, use ... reg, mem32
1802     __ pop_f(v1);
1803     __ fcmps(v1, v0);
1804   } else {
1805     // XXX get rid of pop here, use ... reg, mem64
1806     __ pop_d(v1);
1807     __ fcmpd(v1, v0);
1808   }
1809   if (unordered_result < 0) {
1810     // we want -1 for unordered or less than, 0 for equal and 1 for
1811     // greater than.
1812     __ mov(r0, (u_int64_t)-1L);
1813     // for FP LT tests less than or unordered
1814     __ br(Assembler::LT, done);
1815     // install 0 for EQ otherwise 1
1816     __ csinc(r0, zr, zr, Assembler::EQ);
1817   } else {
1818     // we want -1 for less than, 0 for equal and 1 for unordered or
1819     // greater than.
1820     __ mov(r0, 1L);
1821     // for FP HI tests greater than or unordered
1822     __ br(Assembler::HI, done);
1823     // install 0 for EQ otherwise ~0
1824     __ csinv(r0, zr, zr, Assembler::EQ);
1825 
1826   }
1827   __ bind(done);
1828 }
1829 
1830 void TemplateTable::branch(bool is_jsr, bool is_wide)
1831 {
1832   // We might be moving to a safepoint.  The thread which calls
1833   // Interpreter::notice_safepoints() will effectively flush its cache
1834   // when it makes a system call, but we need to do something to
1835   // ensure that we see the changed dispatch table.
1836   __ membar(MacroAssembler::LoadLoad);
1837 
1838   __ profile_taken_branch(r0, r1);
1839   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1840                              InvocationCounter::counter_offset();
1841   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1842                               InvocationCounter::counter_offset();
1843 
1844   // load branch displacement
1845   if (!is_wide) {
1846     __ ldrh(r2, at_bcp(1));
1847     __ rev16(r2, r2);
1848     // sign extend the 16 bit value in r2
1849     __ sbfm(r2, r2, 0, 15);
1850   } else {
1851     __ ldrw(r2, at_bcp(1));
1852     __ revw(r2, r2);
1853     // sign extend the 32 bit value in r2
1854     __ sbfm(r2, r2, 0, 31);
1855   }
1856 
1857   // Handle all the JSR stuff here, then exit.
1858   // It's much shorter and cleaner than intermingling with the non-JSR
1859   // normal-branch stuff occurring below.
1860 
1861   if (is_jsr) {
1862     // Pre-load the next target bytecode into rscratch1
1863     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1864     // compute return address as bci
1865     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1866     __ add(rscratch2, rscratch2,
1867            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1868     __ sub(r1, rbcp, rscratch2);
1869     __ push_i(r1);
1870     // Adjust the bcp by the 16-bit displacement in r2
1871     __ add(rbcp, rbcp, r2);
1872     __ dispatch_only(vtos, /*generate_poll*/true);
1873     return;
1874   }
1875 
1876   // Normal (non-jsr) branch handling
1877 
1878   // Adjust the bcp by the displacement in r2
1879   __ add(rbcp, rbcp, r2);
1880 
1881   assert(UseLoopCounter || !UseOnStackReplacement,
1882          "on-stack-replacement requires loop counters");
1883   Label backedge_counter_overflow;
1884   Label profile_method;
1885   Label dispatch;
1886   if (UseLoopCounter) {
1887     // increment backedge counter for backward branches
1888     // r0: MDO
1889     // w1: MDO bumped taken-count
1890     // r2: target offset
1891     __ cmp(r2, zr);
1892     __ br(Assembler::GT, dispatch); // count only if backward branch
1893 
1894     // ECN: FIXME: This code smells
1895     // check if MethodCounters exists
1896     Label has_counters;
1897     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1898     __ cbnz(rscratch1, has_counters);
1899     __ push(r0);
1900     __ push(r1);
1901     __ push(r2);
1902     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1903             InterpreterRuntime::build_method_counters), rmethod);
1904     __ pop(r2);
1905     __ pop(r1);
1906     __ pop(r0);
1907     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1908     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1909     __ bind(has_counters);
1910 
1911     if (TieredCompilation) {
1912       Label no_mdo;
1913       int increment = InvocationCounter::count_increment;
1914       if (ProfileInterpreter) {
1915         // Are we profiling?
1916         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1917         __ cbz(r1, no_mdo);
1918         // Increment the MDO backedge counter
1919         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1920                                            in_bytes(InvocationCounter::counter_offset()));
1921         const Address mask(r1, in_bytes(MethodData::backedge_mask_offset()));
1922         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1923                                    r0, rscratch1, false, Assembler::EQ,
1924                                    UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1925         __ b(dispatch);
1926       }
1927       __ bind(no_mdo);
1928       // Increment backedge counter in MethodCounters*
1929       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1930       const Address mask(rscratch1, in_bytes(MethodCounters::backedge_mask_offset()));
1931       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1932                                  r0, rscratch2, false, Assembler::EQ,
1933                                  UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1934     } else { // not TieredCompilation
1935       // increment counter
1936       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1937       __ ldrw(r0, Address(rscratch2, be_offset));        // load backedge counter
1938       __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1939       __ strw(rscratch1, Address(rscratch2, be_offset));        // store counter
1940 
1941       __ ldrw(r0, Address(rscratch2, inv_offset));    // load invocation counter
1942       __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1943       __ addw(r0, r0, rscratch1);        // add both counters
1944 
1945       if (ProfileInterpreter) {
1946         // Test to see if we should create a method data oop
1947         __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_profile_limit_offset())));
1948         __ cmpw(r0, rscratch1);
1949         __ br(Assembler::LT, dispatch);
1950 
1951         // if no method data exists, go to profile method
1952         __ test_method_data_pointer(r0, profile_method);
1953 
1954         if (UseOnStackReplacement) {
1955           // check for overflow against w1 which is the MDO taken count
1956           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1957           __ cmpw(r1, rscratch1);
1958           __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1959 
1960           // When ProfileInterpreter is on, the backedge_count comes
1961           // from the MethodData*, which value does not get reset on
1962           // the call to frequency_counter_overflow().  To avoid
1963           // excessive calls to the overflow routine while the method is
1964           // being compiled, add a second test to make sure the overflow
1965           // function is called only once every overflow_frequency.
1966           const int overflow_frequency = 1024;
1967           __ andsw(r1, r1, overflow_frequency - 1);
1968           __ br(Assembler::EQ, backedge_counter_overflow);
1969 
1970         }
1971       } else {
1972         if (UseOnStackReplacement) {
1973           // check for overflow against w0, which is the sum of the
1974           // counters
1975           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1976           __ cmpw(r0, rscratch1);
1977           __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1978         }
1979       }
1980     }
1981     __ bind(dispatch);
1982   }
1983 
1984   // Pre-load the next target bytecode into rscratch1
1985   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1986 
1987   // continue with the bytecode @ target
1988   // rscratch1: target bytecode
1989   // rbcp: target bcp
1990   __ dispatch_only(vtos, /*generate_poll*/true);
1991 
1992   if (UseLoopCounter) {
1993     if (ProfileInterpreter) {
1994       // Out-of-line code to allocate method data oop.
1995       __ bind(profile_method);
1996       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1997       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1998       __ set_method_data_pointer_for_bcp();
1999       __ b(dispatch);
2000     }
2001 
2002     if (UseOnStackReplacement) {
2003       // invocation counter overflow
2004       __ bind(backedge_counter_overflow);
2005       __ neg(r2, r2);
2006       __ add(r2, r2, rbcp);     // branch bcp
2007       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
2008       __ call_VM(noreg,
2009                  CAST_FROM_FN_PTR(address,
2010                                   InterpreterRuntime::frequency_counter_overflow),
2011                  r2);
2012       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
2013 
2014       // r0: osr nmethod (osr ok) or NULL (osr not possible)
2015       // w1: target bytecode
2016       // r2: scratch
2017       __ cbz(r0, dispatch);     // test result -- no osr if null
2018       // nmethod may have been invalidated (VM may block upon call_VM return)
2019       __ ldrb(r2, Address(r0, nmethod::state_offset()));
2020       if (nmethod::in_use != 0)
2021         __ sub(r2, r2, nmethod::in_use);
2022       __ cbnz(r2, dispatch);
2023 
2024       // We have the address of an on stack replacement routine in r0
2025       // We need to prepare to execute the OSR method. First we must
2026       // migrate the locals and monitors off of the stack.
2027 
2028       __ mov(r19, r0);                             // save the nmethod
2029 
2030       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2031 
2032       // r0 is OSR buffer, move it to expected parameter location
2033       __ mov(j_rarg0, r0);
2034 
2035       // remove activation
2036       // get sender esp
2037       __ ldr(esp,
2038           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
2039       // remove frame anchor
2040       __ leave();
2041       // Ensure compiled code always sees stack at proper alignment
2042       __ andr(sp, esp, -16);
2043 
2044       // and begin the OSR nmethod
2045       __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
2046       __ br(rscratch1);
2047     }
2048   }
2049 }
2050 
2051 
2052 void TemplateTable::if_0cmp(Condition cc)
2053 {
2054   transition(itos, vtos);
2055   // assume branch is more often taken than not (loops use backward branches)
2056   Label not_taken;
2057   if (cc == equal)
2058     __ cbnzw(r0, not_taken);
2059   else if (cc == not_equal)
2060     __ cbzw(r0, not_taken);
2061   else {
2062     __ andsw(zr, r0, r0);
2063     __ br(j_not(cc), not_taken);
2064   }
2065 
2066   branch(false, false);
2067   __ bind(not_taken);
2068   __ profile_not_taken_branch(r0);
2069 }
2070 
2071 void TemplateTable::if_icmp(Condition cc)
2072 {
2073   transition(itos, vtos);
2074   // assume branch is more often taken than not (loops use backward branches)
2075   Label not_taken;
2076   __ pop_i(r1);
2077   __ cmpw(r1, r0, Assembler::LSL);
2078   __ br(j_not(cc), not_taken);
2079   branch(false, false);
2080   __ bind(not_taken);
2081   __ profile_not_taken_branch(r0);
2082 }
2083 
2084 void TemplateTable::if_nullcmp(Condition cc)
2085 {
2086   transition(atos, vtos);
2087   // assume branch is more often taken than not (loops use backward branches)
2088   Label not_taken;
2089   if (cc == equal)
2090     __ cbnz(r0, not_taken);
2091   else
2092     __ cbz(r0, not_taken);
2093   branch(false, false);
2094   __ bind(not_taken);
2095   __ profile_not_taken_branch(r0);
2096 }
2097 
2098 void TemplateTable::if_acmp(Condition cc) {
2099   transition(atos, vtos);
2100   // assume branch is more often taken than not (loops use backward branches)
2101   Label taken, not_taken;
2102   __ pop_ptr(r1);
2103 
2104   Register is_value_mask = rscratch1;
2105   __ mov(is_value_mask, markOopDesc::always_locked_pattern);
2106 
2107   if (EnableValhalla && ACmpOnValues == 3) {
2108     __ cmp(r1, r0);
2109     __ br(Assembler::EQ, (cc == equal) ? taken : not_taken);
2110 
2111     // might be substitutable, test if either r0 or r1 is null
2112     __ andr(r2, r0, r1);
2113     __ cbz(r2, (cc == equal) ? not_taken : taken);
2114 
2115     // and both are values ?
2116     __ ldr(r2, Address(r1, oopDesc::mark_offset_in_bytes()));
2117     __ andr(r2, r2, is_value_mask);
2118     __ ldr(r4, Address(r0, oopDesc::mark_offset_in_bytes()));
2119     __ andr(r4, r4, is_value_mask);
2120     __ andr(r2, r2, r4);
2121     __ cmp(r2,  is_value_mask);
2122     __ br(Assembler::NE, (cc == equal) ? not_taken : taken);
2123 
2124     // same value klass ?
2125     __ load_metadata(r2, r1);
2126     __ load_metadata(r4, r0);
2127     __ cmp(r2, r4);
2128     __ br(Assembler::NE, (cc == equal) ? not_taken : taken);
2129 
2130     // Know both are the same type, let's test for substitutability...
2131     if (cc == equal) {
2132       invoke_is_substitutable(r0, r1, taken, not_taken);
2133     } else {
2134       invoke_is_substitutable(r0, r1, not_taken, taken);
2135     }
2136     __ stop("Not reachable");
2137   }
2138 
2139   if (EnableValhalla && ACmpOnValues == 1) {
2140     Label is_null;
2141     __ cbz(r1, is_null);
2142     __ ldr(r2, Address(r1, oopDesc::mark_offset_in_bytes()));
2143     __ andr(r2, r2, is_value_mask);
2144     __ cmp(r2, is_value_mask);
2145     __ cset(r2, Assembler::EQ);
2146     __ orr(r1, r1, r2);
2147     __ bind(is_null);
2148   }
2149 
2150   __ cmpoop(r1, r0);
2151 
2152   if (EnableValhalla && ACmpOnValues == 2) {
2153     __ br(Assembler::NE, (cc == not_equal) ? taken : not_taken);
2154     __ cbz(r1, (cc == equal) ? taken : not_taken);
2155     __ ldr(r2, Address(r1, oopDesc::mark_offset_in_bytes()));  
2156     __ andr(r2, r2, is_value_mask);
2157     __ cmp(r2, is_value_mask); 
2158     cc = (cc == equal) ? not_equal : equal;
2159   }
2160 
2161   __ br(j_not(cc), not_taken);
2162   __ bind(taken);
2163   branch(false, false);
2164   __ bind(not_taken);
2165   __ profile_not_taken_branch(r0);
2166 }
2167 
2168 void TemplateTable::invoke_is_substitutable(Register aobj, Register bobj,
2169                                             Label& is_subst, Label& not_subst) {
2170 
2171   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::is_substitutable), aobj, bobj);
2172   // Restored... r0 answer, jmp to outcome...
2173   __ cbz(r0, not_subst);
2174   __ b(is_subst);
2175 }
2176 
2177 
2178 void TemplateTable::ret() {
2179   transition(vtos, vtos);
2180   // We might be moving to a safepoint.  The thread which calls
2181   // Interpreter::notice_safepoints() will effectively flush its cache
2182   // when it makes a system call, but we need to do something to
2183   // ensure that we see the changed dispatch table.
2184   __ membar(MacroAssembler::LoadLoad);
2185 
2186   locals_index(r1);
2187   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2188   __ profile_ret(r1, r2);
2189   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2190   __ lea(rbcp, Address(rbcp, r1));
2191   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2192   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2193 }
2194 
2195 void TemplateTable::wide_ret() {
2196   transition(vtos, vtos);
2197   locals_index_wide(r1);
2198   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2199   __ profile_ret(r1, r2);
2200   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2201   __ lea(rbcp, Address(rbcp, r1));
2202   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2203   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2204 }
2205 
2206 
2207 void TemplateTable::tableswitch() {
2208   Label default_case, continue_execution;
2209   transition(itos, vtos);
2210   // align rbcp
2211   __ lea(r1, at_bcp(BytesPerInt));
2212   __ andr(r1, r1, -BytesPerInt);
2213   // load lo & hi
2214   __ ldrw(r2, Address(r1, BytesPerInt));
2215   __ ldrw(r3, Address(r1, 2 * BytesPerInt));
2216   __ rev32(r2, r2);
2217   __ rev32(r3, r3);
2218   // check against lo & hi
2219   __ cmpw(r0, r2);
2220   __ br(Assembler::LT, default_case);
2221   __ cmpw(r0, r3);
2222   __ br(Assembler::GT, default_case);
2223   // lookup dispatch offset
2224   __ subw(r0, r0, r2);
2225   __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2226   __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2227   __ profile_switch_case(r0, r1, r2);
2228   // continue execution
2229   __ bind(continue_execution);
2230   __ rev32(r3, r3);
2231   __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2232   __ add(rbcp, rbcp, r3, ext::sxtw);
2233   __ dispatch_only(vtos, /*generate_poll*/true);
2234   // handle default
2235   __ bind(default_case);
2236   __ profile_switch_default(r0);
2237   __ ldrw(r3, Address(r1, 0));
2238   __ b(continue_execution);
2239 }
2240 
2241 void TemplateTable::lookupswitch() {
2242   transition(itos, itos);
2243   __ stop("lookupswitch bytecode should have been rewritten");
2244 }
2245 
2246 void TemplateTable::fast_linearswitch() {
2247   transition(itos, vtos);
2248   Label loop_entry, loop, found, continue_execution;
2249   // bswap r0 so we can avoid bswapping the table entries
2250   __ rev32(r0, r0);
2251   // align rbcp
2252   __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2253                                     // this instruction (change offsets
2254                                     // below)
2255   __ andr(r19, r19, -BytesPerInt);
2256   // set counter
2257   __ ldrw(r1, Address(r19, BytesPerInt));
2258   __ rev32(r1, r1);
2259   __ b(loop_entry);
2260   // table search
2261   __ bind(loop);
2262   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2263   __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2264   __ cmpw(r0, rscratch1);
2265   __ br(Assembler::EQ, found);
2266   __ bind(loop_entry);
2267   __ subs(r1, r1, 1);
2268   __ br(Assembler::PL, loop);
2269   // default case
2270   __ profile_switch_default(r0);
2271   __ ldrw(r3, Address(r19, 0));
2272   __ b(continue_execution);
2273   // entry found -> get offset
2274   __ bind(found);
2275   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2276   __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2277   __ profile_switch_case(r1, r0, r19);
2278   // continue execution
2279   __ bind(continue_execution);
2280   __ rev32(r3, r3);
2281   __ add(rbcp, rbcp, r3, ext::sxtw);
2282   __ ldrb(rscratch1, Address(rbcp, 0));
2283   __ dispatch_only(vtos, /*generate_poll*/true);
2284 }
2285 
2286 void TemplateTable::fast_binaryswitch() {
2287   transition(itos, vtos);
2288   // Implementation using the following core algorithm:
2289   //
2290   // int binary_search(int key, LookupswitchPair* array, int n) {
2291   //   // Binary search according to "Methodik des Programmierens" by
2292   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2293   //   int i = 0;
2294   //   int j = n;
2295   //   while (i+1 < j) {
2296   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2297   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2298   //     // where a stands for the array and assuming that the (inexisting)
2299   //     // element a[n] is infinitely big.
2300   //     int h = (i + j) >> 1;
2301   //     // i < h < j
2302   //     if (key < array[h].fast_match()) {
2303   //       j = h;
2304   //     } else {
2305   //       i = h;
2306   //     }
2307   //   }
2308   //   // R: a[i] <= key < a[i+1] or Q
2309   //   // (i.e., if key is within array, i is the correct index)
2310   //   return i;
2311   // }
2312 
2313   // Register allocation
2314   const Register key   = r0; // already set (tosca)
2315   const Register array = r1;
2316   const Register i     = r2;
2317   const Register j     = r3;
2318   const Register h     = rscratch1;
2319   const Register temp  = rscratch2;
2320 
2321   // Find array start
2322   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2323                                           // get rid of this
2324                                           // instruction (change
2325                                           // offsets below)
2326   __ andr(array, array, -BytesPerInt);
2327 
2328   // Initialize i & j
2329   __ mov(i, 0);                            // i = 0;
2330   __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2331 
2332   // Convert j into native byteordering
2333   __ rev32(j, j);
2334 
2335   // And start
2336   Label entry;
2337   __ b(entry);
2338 
2339   // binary search loop
2340   {
2341     Label loop;
2342     __ bind(loop);
2343     // int h = (i + j) >> 1;
2344     __ addw(h, i, j);                           // h = i + j;
2345     __ lsrw(h, h, 1);                                   // h = (i + j) >> 1;
2346     // if (key < array[h].fast_match()) {
2347     //   j = h;
2348     // } else {
2349     //   i = h;
2350     // }
2351     // Convert array[h].match to native byte-ordering before compare
2352     __ ldr(temp, Address(array, h, Address::lsl(3)));
2353     __ rev32(temp, temp);
2354     __ cmpw(key, temp);
2355     // j = h if (key <  array[h].fast_match())
2356     __ csel(j, h, j, Assembler::LT);
2357     // i = h if (key >= array[h].fast_match())
2358     __ csel(i, h, i, Assembler::GE);
2359     // while (i+1 < j)
2360     __ bind(entry);
2361     __ addw(h, i, 1);          // i+1
2362     __ cmpw(h, j);             // i+1 < j
2363     __ br(Assembler::LT, loop);
2364   }
2365 
2366   // end of binary search, result index is i (must check again!)
2367   Label default_case;
2368   // Convert array[i].match to native byte-ordering before compare
2369   __ ldr(temp, Address(array, i, Address::lsl(3)));
2370   __ rev32(temp, temp);
2371   __ cmpw(key, temp);
2372   __ br(Assembler::NE, default_case);
2373 
2374   // entry found -> j = offset
2375   __ add(j, array, i, ext::uxtx, 3);
2376   __ ldrw(j, Address(j, BytesPerInt));
2377   __ profile_switch_case(i, key, array);
2378   __ rev32(j, j);
2379   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2380   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2381   __ dispatch_only(vtos, /*generate_poll*/true);
2382 
2383   // default case -> j = default offset
2384   __ bind(default_case);
2385   __ profile_switch_default(i);
2386   __ ldrw(j, Address(array, -2 * BytesPerInt));
2387   __ rev32(j, j);
2388   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2389   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2390   __ dispatch_only(vtos, /*generate_poll*/true);
2391 }
2392 
2393 
2394 void TemplateTable::_return(TosState state)
2395 {
2396   transition(state, state);
2397   assert(_desc->calls_vm(),
2398          "inconsistent calls_vm information"); // call in remove_activation
2399 
2400   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2401     assert(state == vtos, "only valid state");
2402 
2403     __ ldr(c_rarg1, aaddress(0));
2404     __ load_klass(r3, c_rarg1);
2405     __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2406     Label skip_register_finalizer;
2407     __ tbz(r3, exact_log2(JVM_ACC_HAS_FINALIZER), skip_register_finalizer);
2408 
2409     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2410 
2411     __ bind(skip_register_finalizer);
2412   }
2413 
2414   // Issue a StoreStore barrier after all stores but before return
2415   // from any constructor for any class with a final field.  We don't
2416   // know if this is a finalizer, so we always do so.
2417   if (_desc->bytecode() == Bytecodes::_return)
2418     __ membar(MacroAssembler::StoreStore);
2419 
2420   // Narrow result if state is itos but result type is smaller.
2421   // Need to narrow in the return bytecode rather than in generate_return_entry
2422   // since compiled code callers expect the result to already be narrowed.
2423   if (state == itos) {
2424     __ narrow(r0);
2425   }
2426 
2427   __ remove_activation(state); 
2428   __ ret(lr);
2429 }
2430 
2431 // ----------------------------------------------------------------------------
2432 // Volatile variables demand their effects be made known to all CPU's
2433 // in order.  Store buffers on most chips allow reads & writes to
2434 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2435 // without some kind of memory barrier (i.e., it's not sufficient that
2436 // the interpreter does not reorder volatile references, the hardware
2437 // also must not reorder them).
2438 //
2439 // According to the new Java Memory Model (JMM):
2440 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2441 //     writes act as aquire & release, so:
2442 // (2) A read cannot let unrelated NON-volatile memory refs that
2443 //     happen after the read float up to before the read.  It's OK for
2444 //     non-volatile memory refs that happen before the volatile read to
2445 //     float down below it.
2446 // (3) Similar a volatile write cannot let unrelated NON-volatile
2447 //     memory refs that happen BEFORE the write float down to after the
2448 //     write.  It's OK for non-volatile memory refs that happen after the
2449 //     volatile write to float up before it.
2450 //
2451 // We only put in barriers around volatile refs (they are expensive),
2452 // not _between_ memory refs (that would require us to track the
2453 // flavor of the previous memory refs).  Requirements (2) and (3)
2454 // require some barriers before volatile stores and after volatile
2455 // loads.  These nearly cover requirement (1) but miss the
2456 // volatile-store-volatile-load case.  This final case is placed after
2457 // volatile-stores although it could just as well go before
2458 // volatile-loads.
2459 
2460 void TemplateTable::resolve_cache_and_index(int byte_no,
2461                                             Register Rcache,
2462                                             Register index,
2463                                             size_t index_size) {
2464   const Register temp = r19;
2465   assert_different_registers(Rcache, index, temp);
2466 
2467   Label resolved;
2468 
2469   Bytecodes::Code code = bytecode();
2470   switch (code) {
2471   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2472   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2473   default: break;
2474   }
2475 
2476   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2477   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2478   __ subs(zr, temp, (int) code);  // have we resolved this bytecode?
2479   __ br(Assembler::EQ, resolved);
2480 
2481   // resolve first time through
2482   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2483   __ mov(temp, (int) code);
2484   __ call_VM(noreg, entry, temp);
2485 
2486   // Update registers with resolved info
2487   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2488   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2489   // so all clients ofthis method must be modified accordingly
2490   __ bind(resolved);
2491 }
2492 
2493 // The Rcache and index registers must be set before call
2494 // n.b unlike x86 cache already includes the index offset
2495 void TemplateTable::load_field_cp_cache_entry(Register obj,
2496                                               Register cache,
2497                                               Register index,
2498                                               Register off,
2499                                               Register flags,
2500                                               bool is_static = false) {
2501   assert_different_registers(cache, index, flags, off);
2502 
2503   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2504   // Field offset
2505   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2506                                           ConstantPoolCacheEntry::f2_offset())));
2507   // Flags
2508   __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2509                                            ConstantPoolCacheEntry::flags_offset())));
2510 
2511   // klass overwrite register
2512   if (is_static) {
2513     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2514                                         ConstantPoolCacheEntry::f1_offset())));
2515     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2516     __ ldr(obj, Address(obj, mirror_offset));
2517     __ resolve_oop_handle(obj);
2518   }
2519 }
2520 
2521 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2522                                                Register method,
2523                                                Register itable_index,
2524                                                Register flags,
2525                                                bool is_invokevirtual,
2526                                                bool is_invokevfinal, /*unused*/
2527                                                bool is_invokedynamic) {
2528   // setup registers
2529   const Register cache = rscratch2;
2530   const Register index = r4;
2531   assert_different_registers(method, flags);
2532   assert_different_registers(method, cache, index);
2533   assert_different_registers(itable_index, flags);
2534   assert_different_registers(itable_index, cache, index);
2535   // determine constant pool cache field offsets
2536   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2537   const int method_offset = in_bytes(
2538     ConstantPoolCache::base_offset() +
2539       (is_invokevirtual
2540        ? ConstantPoolCacheEntry::f2_offset()
2541        : ConstantPoolCacheEntry::f1_offset()));
2542   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2543                                     ConstantPoolCacheEntry::flags_offset());
2544   // access constant pool cache fields
2545   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2546                                     ConstantPoolCacheEntry::f2_offset());
2547 
2548   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2549   resolve_cache_and_index(byte_no, cache, index, index_size);
2550   __ ldr(method, Address(cache, method_offset));
2551 
2552   if (itable_index != noreg) {
2553     __ ldr(itable_index, Address(cache, index_offset));
2554   }
2555   __ ldrw(flags, Address(cache, flags_offset));
2556 }
2557 
2558 
2559 // The registers cache and index expected to be set before call.
2560 // Correct values of the cache and index registers are preserved.
2561 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2562                                             bool is_static, bool has_tos) {
2563   // do the JVMTI work here to avoid disturbing the register state below
2564   // We use c_rarg registers here because we want to use the register used in
2565   // the call to the VM
2566   if (JvmtiExport::can_post_field_access()) {
2567     // Check to see if a field access watch has been set before we
2568     // take the time to call into the VM.
2569     Label L1;
2570     assert_different_registers(cache, index, r0);
2571     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2572     __ ldrw(r0, Address(rscratch1));
2573     __ cbzw(r0, L1);
2574 
2575     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2576     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2577 
2578     if (is_static) {
2579       __ mov(c_rarg1, zr); // NULL object reference
2580     } else {
2581       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2582       __ verify_oop(c_rarg1);
2583     }
2584     // c_rarg1: object pointer or NULL
2585     // c_rarg2: cache entry pointer
2586     // c_rarg3: jvalue object on the stack
2587     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2588                                        InterpreterRuntime::post_field_access),
2589                c_rarg1, c_rarg2, c_rarg3);
2590     __ get_cache_and_index_at_bcp(cache, index, 1);
2591     __ bind(L1);
2592   }
2593 }
2594 
2595 void TemplateTable::pop_and_check_object(Register r)
2596 {
2597   __ pop_ptr(r);
2598   __ null_check(r);  // for field access must check obj.
2599   __ verify_oop(r);
2600 }
2601 
2602 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2603 {
2604   const Register cache = r2;
2605   const Register index = r3;
2606   const Register obj   = r4;
2607   const Register off   = r19;
2608   const Register flags = r0;
2609   const Register raw_flags = r6;
2610   const Register bc    = r4; // uses same reg as obj, so don't mix them
2611 
2612   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2613   jvmti_post_field_access(cache, index, is_static, false);
2614   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2615 
2616   if (!is_static) {
2617     // obj is on the stack
2618     pop_and_check_object(obj);
2619   }
2620 
2621   // 8179954: We need to make sure that the code generated for
2622   // volatile accesses forms a sequentially-consistent set of
2623   // operations when combined with STLR and LDAR.  Without a leading
2624   // membar it's possible for a simple Dekker test to fail if loads
2625   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2626   // the stores in one method and we interpret the loads in another.
2627   if (! UseBarriersForVolatile) {
2628     Label notVolatile;
2629     __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2630     __ membar(MacroAssembler::AnyAny);
2631     __ bind(notVolatile);
2632   }
2633 
2634   const Address field(obj, off);
2635 
2636   Label Done, notByte, notBool, notInt, notShort, notChar,
2637               notLong, notFloat, notObj, notDouble;
2638 
2639   // x86 uses a shift and mask or wings it with a shift plus assert
2640   // the mask is not needed. aarch64 just uses bitfield extract
2641   __ ubfxw(flags, raw_flags, ConstantPoolCacheEntry::tos_state_shift, ConstantPoolCacheEntry::tos_state_bits);
2642 
2643   assert(btos == 0, "change code, btos != 0");
2644   __ cbnz(flags, notByte);
2645 
2646   // Don't rewrite getstatic, only getfield
2647   if (is_static) rc = may_not_rewrite;
2648 
2649   // btos
2650   __ access_load_at(T_BYTE, IN_HEAP, r0, field, noreg, noreg);
2651   __ push(btos);
2652   // Rewrite bytecode to be faster
2653   if (rc == may_rewrite) {
2654     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2655   }
2656   __ b(Done);
2657 
2658   __ bind(notByte);
2659   __ cmp(flags, (u1)ztos);
2660   __ br(Assembler::NE, notBool);
2661 
2662   // ztos (same code as btos)
2663   __ access_load_at(T_BOOLEAN, IN_HEAP, r0, field, noreg, noreg);
2664   __ push(ztos);
2665   // Rewrite bytecode to be faster
2666   if (rc == may_rewrite) {
2667     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2668     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2669   }
2670   __ b(Done);
2671 
2672   __ bind(notBool);
2673   __ cmp(flags, (u1)atos);
2674   __ br(Assembler::NE, notObj);
2675   // atos
2676   if (!EnableValhalla) {
2677     do_oop_load(_masm, field, r0, IN_HEAP);
2678     __ push(atos);
2679     if (rc == may_rewrite) {
2680       patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2681     }  
2682     __ b(Done);
2683   } else { // Valhalla
2684 
2685     if (is_static) {
2686       __ load_heap_oop(r0, field);
2687       Label isFlattenable, isUninitialized;
2688       // Issue below if the static field has not been initialized yet
2689       __ test_field_is_flattenable(raw_flags, r8 /*temp*/, isFlattenable);
2690         // Not flattenable case
2691         __ push(atos);
2692         __ b(Done);
2693       // Flattenable case, must not return null even if uninitialized
2694       __ bind(isFlattenable);
2695         __ cbz(r0, isUninitialized);
2696           __ push(atos);
2697           __ b(Done);
2698         __ bind(isUninitialized);
2699           __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2700           __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_value_field), obj, raw_flags);
2701           __ verify_oop(r0);
2702           __ push(atos);
2703           __ b(Done);
2704     } else {
2705       Label isFlattened, isInitialized, isFlattenable, rewriteFlattenable;
2706         __ test_field_is_flattenable(raw_flags, r8 /*temp*/, isFlattenable);
2707         // Non-flattenable field case, also covers the object case
2708         __ load_heap_oop(r0, field);
2709         __ push(atos);
2710         if (rc == may_rewrite) {
2711           patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2712         }
2713         __ b(Done);
2714       __ bind(isFlattenable);
2715         __ test_field_is_flattened(raw_flags, r8 /* temp */, isFlattened);
2716          // Non-flattened field case
2717           __ load_heap_oop(r0, field);
2718           __ cbnz(r0, isInitialized);
2719             __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2720             __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_instance_value_field), obj, raw_flags);
2721           __ bind(isInitialized);
2722           __ verify_oop(r0);
2723           __ push(atos);
2724           __ b(rewriteFlattenable);
2725         __ bind(isFlattened);
2726           __ ldr(r10, Address(cache, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset())));
2727           __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2728           call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), obj, raw_flags, r10); 
2729           __ verify_oop(r0);
2730           __ push(atos);
2731       __ bind(rewriteFlattenable);
2732       if (rc == may_rewrite) { 
2733          patch_bytecode(Bytecodes::_fast_qgetfield, bc, r1);
2734       }
2735       __ b(Done);
2736     }
2737   }
2738 
2739   __ bind(notObj);
2740   __ cmp(flags, (u1)itos);
2741   __ br(Assembler::NE, notInt);
2742   // itos
2743   __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
2744   __ push(itos);
2745   // Rewrite bytecode to be faster
2746   if (rc == may_rewrite) {
2747     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2748   }
2749   __ b(Done);
2750 
2751   __ bind(notInt);
2752   __ cmp(flags, (u1)ctos);
2753   __ br(Assembler::NE, notChar);
2754   // ctos
2755   __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
2756   __ push(ctos);
2757   // Rewrite bytecode to be faster
2758   if (rc == may_rewrite) {
2759     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2760   }
2761   __ b(Done);
2762 
2763   __ bind(notChar);
2764   __ cmp(flags, (u1)stos);
2765   __ br(Assembler::NE, notShort);
2766   // stos
2767   __ access_load_at(T_SHORT, IN_HEAP, r0, field, noreg, noreg);
2768   __ push(stos);
2769   // Rewrite bytecode to be faster
2770   if (rc == may_rewrite) {
2771     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2772   }
2773   __ b(Done);
2774 
2775   __ bind(notShort);
2776   __ cmp(flags, (u1)ltos);
2777   __ br(Assembler::NE, notLong);
2778   // ltos
2779   __ access_load_at(T_LONG, IN_HEAP, r0, field, noreg, noreg);
2780   __ push(ltos);
2781   // Rewrite bytecode to be faster
2782   if (rc == may_rewrite) {
2783     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2784   }
2785   __ b(Done);
2786 
2787   __ bind(notLong);
2788   __ cmp(flags, (u1)ftos);
2789   __ br(Assembler::NE, notFloat);
2790   // ftos
2791   __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2792   __ push(ftos);
2793   // Rewrite bytecode to be faster
2794   if (rc == may_rewrite) {
2795     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2796   }
2797   __ b(Done);
2798 
2799   __ bind(notFloat);
2800 #ifdef ASSERT
2801   __ cmp(flags, (u1)dtos);
2802   __ br(Assembler::NE, notDouble);
2803 #endif
2804   // dtos
2805   __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2806   __ push(dtos);
2807   // Rewrite bytecode to be faster
2808   if (rc == may_rewrite) {
2809     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2810   }
2811 #ifdef ASSERT
2812   __ b(Done);
2813 
2814   __ bind(notDouble);
2815   __ stop("Bad state");
2816 #endif
2817 
2818   __ bind(Done);
2819 
2820   Label notVolatile;
2821   __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2822   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2823   __ bind(notVolatile);
2824 }
2825 
2826 
2827 void TemplateTable::getfield(int byte_no)
2828 {
2829   getfield_or_static(byte_no, false);
2830 }
2831 
2832 void TemplateTable::nofast_getfield(int byte_no) {
2833   getfield_or_static(byte_no, false, may_not_rewrite);
2834 }
2835 
2836 void TemplateTable::getstatic(int byte_no)
2837 {
2838   getfield_or_static(byte_no, true);
2839 }
2840 
2841 // The registers cache and index expected to be set before call.
2842 // The function may destroy various registers, just not the cache and index registers.
2843 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2844   transition(vtos, vtos);
2845 
2846   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2847 
2848   if (JvmtiExport::can_post_field_modification()) {
2849     // Check to see if a field modification watch has been set before
2850     // we take the time to call into the VM.
2851     Label L1;
2852     assert_different_registers(cache, index, r0);
2853     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2854     __ ldrw(r0, Address(rscratch1));
2855     __ cbz(r0, L1);
2856 
2857     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2858 
2859     if (is_static) {
2860       // Life is simple.  Null out the object pointer.
2861       __ mov(c_rarg1, zr);
2862     } else {
2863       // Life is harder. The stack holds the value on top, followed by
2864       // the object.  We don't know the size of the value, though; it
2865       // could be one or two words depending on its type. As a result,
2866       // we must find the type to determine where the object is.
2867       __ ldrw(c_rarg3, Address(c_rarg2,
2868                                in_bytes(cp_base_offset +
2869                                         ConstantPoolCacheEntry::flags_offset())));
2870       __ lsr(c_rarg3, c_rarg3,
2871              ConstantPoolCacheEntry::tos_state_shift);
2872       ConstantPoolCacheEntry::verify_tos_state_shift();
2873       Label nope2, done, ok;
2874       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2875       __ cmpw(c_rarg3, ltos);
2876       __ br(Assembler::EQ, ok);
2877       __ cmpw(c_rarg3, dtos);
2878       __ br(Assembler::NE, nope2);
2879       __ bind(ok);
2880       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2881       __ bind(nope2);
2882     }
2883     // cache entry pointer
2884     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2885     // object (tos)
2886     __ mov(c_rarg3, esp);
2887     // c_rarg1: object pointer set up above (NULL if static)
2888     // c_rarg2: cache entry pointer
2889     // c_rarg3: jvalue object on the stack
2890     __ call_VM(noreg,
2891                CAST_FROM_FN_PTR(address,
2892                                 InterpreterRuntime::post_field_modification),
2893                c_rarg1, c_rarg2, c_rarg3);
2894     __ get_cache_and_index_at_bcp(cache, index, 1);
2895     __ bind(L1);
2896   }
2897 }
2898 
2899 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2900   transition(vtos, vtos);
2901 
2902   const Register cache = r2;
2903   const Register index = r3;
2904   const Register obj   = r2;
2905   const Register off   = r19;
2906   const Register flags = r0;
2907   const Register flags2 = r6;
2908   const Register bc    = r4;
2909 
2910   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2911   jvmti_post_field_mod(cache, index, is_static);
2912   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2913 
2914   Label Done;
2915   __ mov(r5, flags);
2916 
2917   {
2918     Label notVolatile;
2919     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2920     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2921     __ bind(notVolatile);
2922   }
2923 
2924   // field address
2925   const Address field(obj, off);
2926 
2927   Label notByte, notBool, notInt, notShort, notChar,
2928         notLong, notFloat, notObj, notDouble;
2929 
2930   __ mov(flags2, flags); 
2931 
2932   // x86 uses a shift and mask or wings it with a shift plus assert
2933   // the mask is not needed. aarch64 just uses bitfield extract
2934   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2935 
2936   assert(btos == 0, "change code, btos != 0");
2937   __ cbnz(flags, notByte);
2938 
2939   // Don't rewrite putstatic, only putfield
2940   if (is_static) rc = may_not_rewrite;
2941 
2942   // btos
2943   {
2944     __ pop(btos);
2945     if (!is_static) pop_and_check_object(obj);
2946     __ access_store_at(T_BYTE, IN_HEAP, field, r0, noreg, noreg);
2947     if (rc == may_rewrite) {
2948       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2949     }
2950     __ b(Done);
2951   }
2952 
2953   __ bind(notByte);
2954   __ cmp(flags, (u1)ztos);
2955   __ br(Assembler::NE, notBool);
2956 
2957   // ztos
2958   {
2959     __ pop(ztos);
2960     if (!is_static) pop_and_check_object(obj);
2961     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg);
2962     if (rc == may_rewrite) {
2963       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2964     }
2965     __ b(Done);
2966   }
2967 
2968   __ bind(notBool);
2969   __ cmp(flags, (u1)atos);
2970   __ br(Assembler::NE, notObj);
2971 
2972   // atos
2973   {
2974      if (!EnableValhalla) {
2975       __ pop(atos);
2976       if (!is_static) pop_and_check_object(obj);
2977       // Store into the field
2978       do_oop_store(_masm, field, r0, IN_HEAP);
2979       if (rc == may_rewrite) {
2980         patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2981       }
2982       __ b(Done);
2983      } else { // Valhalla
2984 
2985       __ pop(atos);
2986       if (is_static) {
2987         Label notFlattenable;
2988          __ test_field_is_not_flattenable(flags2, r8 /* temp */, notFlattenable);
2989          __ null_check(r0);
2990          __ bind(notFlattenable);
2991          do_oop_store(_masm, field, r0, IN_HEAP); 
2992          __ b(Done);
2993       } else {
2994         Label isFlattenable, isFlattened, notBuffered, notBuffered2, rewriteNotFlattenable, rewriteFlattenable;
2995         __ test_field_is_flattenable(flags2, r8 /*temp*/, isFlattenable);
2996         // Not flattenable case, covers not flattenable values and objects
2997         pop_and_check_object(obj);
2998         // Store into the field
2999         do_oop_store(_masm, field, r0, IN_HEAP);
3000         __ bind(rewriteNotFlattenable);
3001         if (rc == may_rewrite) {
3002           patch_bytecode(Bytecodes::_fast_aputfield, bc, r19, true, byte_no); 
3003         }
3004         __ b(Done);
3005         // Implementation of the flattenable semantic
3006         __ bind(isFlattenable);
3007         __ null_check(r0);
3008         __ test_field_is_flattened(flags2, r8 /*temp*/, isFlattened);
3009         // Not flattened case
3010         pop_and_check_object(obj);
3011         // Store into the field
3012         do_oop_store(_masm, field, r0, IN_HEAP);
3013         __ b(rewriteFlattenable);
3014         __ bind(isFlattened);
3015         pop_and_check_object(obj);
3016         call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flattened_value), r0, off, obj);
3017         __ bind(rewriteFlattenable);
3018         if (rc == may_rewrite) {
3019           patch_bytecode(Bytecodes::_fast_qputfield, bc, r19, true, byte_no);
3020         }
3021         __ b(Done);
3022       }
3023      }  // Valhalla
3024   }
3025 
3026   __ bind(notObj);
3027   __ cmp(flags, (u1)itos);
3028   __ br(Assembler::NE, notInt);
3029 
3030   // itos
3031   {
3032     __ pop(itos);
3033     if (!is_static) pop_and_check_object(obj);
3034     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg);
3035     if (rc == may_rewrite) {
3036       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
3037     }
3038     __ b(Done);
3039   }
3040 
3041   __ bind(notInt);
3042   __ cmp(flags, (u1)ctos);
3043   __ br(Assembler::NE, notChar);
3044 
3045   // ctos
3046   {
3047     __ pop(ctos);
3048     if (!is_static) pop_and_check_object(obj);
3049     __ access_store_at(T_CHAR, IN_HEAP, field, r0, noreg, noreg);
3050     if (rc == may_rewrite) {
3051       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
3052     }
3053     __ b(Done);
3054   }
3055 
3056   __ bind(notChar);
3057   __ cmp(flags, (u1)stos);
3058   __ br(Assembler::NE, notShort);
3059 
3060   // stos
3061   {
3062     __ pop(stos);
3063     if (!is_static) pop_and_check_object(obj);
3064     __ access_store_at(T_SHORT, IN_HEAP, field, r0, noreg, noreg);
3065     if (rc == may_rewrite) {
3066       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
3067     }
3068     __ b(Done);
3069   }
3070 
3071   __ bind(notShort);
3072   __ cmp(flags, (u1)ltos);
3073   __ br(Assembler::NE, notLong);
3074 
3075   // ltos
3076   {
3077     __ pop(ltos);
3078     if (!is_static) pop_and_check_object(obj);
3079     __ access_store_at(T_LONG, IN_HEAP, field, r0, noreg, noreg);
3080     if (rc == may_rewrite) {
3081       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
3082     }
3083     __ b(Done);
3084   }
3085 
3086   __ bind(notLong);
3087   __ cmp(flags, (u1)ftos);
3088   __ br(Assembler::NE, notFloat);
3089 
3090   // ftos
3091   {
3092     __ pop(ftos);
3093     if (!is_static) pop_and_check_object(obj);
3094     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
3095     if (rc == may_rewrite) {
3096       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
3097     }
3098     __ b(Done);
3099   }
3100 
3101   __ bind(notFloat);
3102 #ifdef ASSERT
3103   __ cmp(flags, (u1)dtos);
3104   __ br(Assembler::NE, notDouble);
3105 #endif
3106 
3107   // dtos
3108   {
3109     __ pop(dtos);
3110     if (!is_static) pop_and_check_object(obj);
3111     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
3112     if (rc == may_rewrite) {
3113       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
3114     }
3115   }
3116 
3117 #ifdef ASSERT
3118   __ b(Done);
3119 
3120   __ bind(notDouble);
3121   __ stop("Bad state");
3122 #endif
3123 
3124   __ bind(Done);
3125 
3126   {
3127     Label notVolatile;
3128     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3129     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
3130     __ bind(notVolatile);
3131   }
3132 }
3133 
3134 void TemplateTable::putfield(int byte_no)
3135 {
3136   putfield_or_static(byte_no, false);
3137 }
3138 
3139 void TemplateTable::nofast_putfield(int byte_no) {
3140   putfield_or_static(byte_no, false, may_not_rewrite);
3141 }
3142 
3143 void TemplateTable::putstatic(int byte_no) {
3144   putfield_or_static(byte_no, true);
3145 }
3146 
3147 void TemplateTable::jvmti_post_fast_field_mod()
3148 {
3149   if (JvmtiExport::can_post_field_modification()) {
3150     // Check to see if a field modification watch has been set before
3151     // we take the time to call into the VM.
3152     Label L2;
3153     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3154     __ ldrw(c_rarg3, Address(rscratch1));
3155     __ cbzw(c_rarg3, L2);
3156     __ pop_ptr(r19);                  // copy the object pointer from tos
3157     __ verify_oop(r19);
3158     __ push_ptr(r19);                 // put the object pointer back on tos
3159     // Save tos values before call_VM() clobbers them. Since we have
3160     // to do it for every data type, we use the saved values as the
3161     // jvalue object.
3162     switch (bytecode()) {          // load values into the jvalue object
3163     case Bytecodes::_fast_qputfield: //fall through
3164     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
3165     case Bytecodes::_fast_bputfield: // fall through
3166     case Bytecodes::_fast_zputfield: // fall through
3167     case Bytecodes::_fast_sputfield: // fall through
3168     case Bytecodes::_fast_cputfield: // fall through
3169     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
3170     case Bytecodes::_fast_dputfield: __ push_d(); break;
3171     case Bytecodes::_fast_fputfield: __ push_f(); break;
3172     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
3173 
3174     default:
3175       ShouldNotReachHere();
3176     }
3177     __ mov(c_rarg3, esp);             // points to jvalue on the stack
3178     // access constant pool cache entry
3179     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
3180     __ verify_oop(r19);
3181     // r19: object pointer copied above
3182     // c_rarg2: cache entry pointer
3183     // c_rarg3: jvalue object on the stack
3184     __ call_VM(noreg,
3185                CAST_FROM_FN_PTR(address,
3186                                 InterpreterRuntime::post_field_modification),
3187                r19, c_rarg2, c_rarg3);
3188 
3189     switch (bytecode()) {             // restore tos values
3190     case Bytecodes::_fast_qputfield: //fall through
3191     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
3192     case Bytecodes::_fast_bputfield: // fall through
3193     case Bytecodes::_fast_zputfield: // fall through
3194     case Bytecodes::_fast_sputfield: // fall through
3195     case Bytecodes::_fast_cputfield: // fall through
3196     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
3197     case Bytecodes::_fast_dputfield: __ pop_d(); break;
3198     case Bytecodes::_fast_fputfield: __ pop_f(); break;
3199     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
3200     default: break;
3201     }
3202     __ bind(L2);
3203   }
3204 }
3205 
3206 void TemplateTable::fast_storefield(TosState state)
3207 {
3208   transition(state, vtos);
3209 
3210   ByteSize base = ConstantPoolCache::base_offset();
3211 
3212   jvmti_post_fast_field_mod();
3213 
3214   // access constant pool cache
3215   __ get_cache_and_index_at_bcp(r2, r1, 1);
3216 
3217   // test for volatile with r3
3218   __ ldrw(r3, Address(r2, in_bytes(base +
3219                                    ConstantPoolCacheEntry::flags_offset())));
3220 
3221   // replace index with field offset from cache entry
3222   __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
3223 
3224   {
3225     Label notVolatile;
3226     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3227     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
3228     __ bind(notVolatile);
3229   }
3230 
3231   Label notVolatile;
3232 
3233   // Get object from stack
3234   pop_and_check_object(r2);
3235 
3236   // field address
3237   const Address field(r2, r1);
3238 
3239   // access field
3240   switch (bytecode()) {
3241   case Bytecodes::_fast_qputfield: //fall through 
3242    {
3243       Label isFlattened, done; 
3244       __ null_check(r0);
3245       __ test_field_is_flattened(r3, r8 /* temp */, isFlattened);
3246       // No Flattened case
3247       do_oop_store(_masm, field, r0, IN_HEAP);
3248       __ b(done);
3249       __ bind(isFlattened);
3250       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flattened_value), r0, r1, r2);
3251       __ bind(done);
3252     }
3253     break;
3254   case Bytecodes::_fast_aputfield:
3255     do_oop_store(_masm, field, r0, IN_HEAP);
3256     break;
3257   case Bytecodes::_fast_lputfield:
3258     __ access_store_at(T_LONG, IN_HEAP, field, r0, noreg, noreg);
3259     break;
3260   case Bytecodes::_fast_iputfield:
3261     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg);
3262     break;
3263   case Bytecodes::_fast_zputfield:
3264     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg);
3265     break;
3266   case Bytecodes::_fast_bputfield:
3267     __ access_store_at(T_BYTE, IN_HEAP, field, r0, noreg, noreg);
3268     break;
3269   case Bytecodes::_fast_sputfield:
3270     __ access_store_at(T_SHORT, IN_HEAP, field, r0, noreg, noreg);
3271     break;
3272   case Bytecodes::_fast_cputfield:
3273     __ access_store_at(T_CHAR, IN_HEAP, field, r0, noreg, noreg);
3274     break;
3275   case Bytecodes::_fast_fputfield:
3276     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
3277     break;
3278   case Bytecodes::_fast_dputfield:
3279     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
3280     break;
3281   default:
3282     ShouldNotReachHere();
3283   }
3284 
3285   {
3286     Label notVolatile;
3287     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3288     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
3289     __ bind(notVolatile);
3290   }
3291 }
3292 
3293 
3294 void TemplateTable::fast_accessfield(TosState state)
3295 {
3296   transition(atos, state);
3297   // Do the JVMTI work here to avoid disturbing the register state below
3298   if (JvmtiExport::can_post_field_access()) {
3299     // Check to see if a field access watch has been set before we
3300     // take the time to call into the VM.
3301     Label L1;
3302     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3303     __ ldrw(r2, Address(rscratch1));
3304     __ cbzw(r2, L1);
3305     // access constant pool cache entry
3306     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
3307     __ verify_oop(r0);
3308     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
3309     __ mov(c_rarg1, r0);
3310     // c_rarg1: object pointer copied above
3311     // c_rarg2: cache entry pointer
3312     __ call_VM(noreg,
3313                CAST_FROM_FN_PTR(address,
3314                                 InterpreterRuntime::post_field_access),
3315                c_rarg1, c_rarg2);
3316     __ pop_ptr(r0); // restore object pointer
3317     __ bind(L1);
3318   }
3319 
3320   // access constant pool cache
3321   __ get_cache_and_index_at_bcp(r2, r1, 1);
3322   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3323                                   ConstantPoolCacheEntry::f2_offset())));
3324   __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3325                                    ConstantPoolCacheEntry::flags_offset())));
3326 
3327   // r0: object
3328   __ verify_oop(r0);
3329   __ null_check(r0);
3330   const Address field(r0, r1);
3331 
3332   // 8179954: We need to make sure that the code generated for
3333   // volatile accesses forms a sequentially-consistent set of
3334   // operations when combined with STLR and LDAR.  Without a leading
3335   // membar it's possible for a simple Dekker test to fail if loads
3336   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3337   // the stores in one method and we interpret the loads in another.
3338   if (! UseBarriersForVolatile) {
3339     Label notVolatile;
3340     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3341     __ membar(MacroAssembler::AnyAny);
3342     __ bind(notVolatile);
3343   }
3344 
3345   // access field
3346   switch (bytecode()) {
3347   case Bytecodes::_fast_qgetfield: 
3348     {
3349        Label isFlattened, isInitialized, Done;
3350        // DMS CHECK: We don't need to reload multiple times, but stay close to original code
3351        __ ldrw(r9, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()))); 
3352        __ test_field_is_flattened(r9, r8 /* temp */, isFlattened);
3353         // Non-flattened field case
3354         __ mov(r9, r0);
3355         __ load_heap_oop(r0, field);
3356         __ cbnz(r0, isInitialized);
3357           __ mov(r0, r9);
3358           __ ldrw(r9, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()))); 
3359           __ andw(r9, r9, ConstantPoolCacheEntry::field_index_mask);
3360           __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_instance_value_field), r0, r9);
3361         __ bind(isInitialized);
3362         __ verify_oop(r0);
3363         __ b(Done);
3364       __ bind(isFlattened);
3365         __ ldrw(r9, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset())));
3366         __ andw(r9, r9, ConstantPoolCacheEntry::field_index_mask);
3367         __ ldr(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset())));
3368         call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), r0, r9, r3);
3369         __ verify_oop(r0);
3370       __ bind(Done);
3371     }
3372     break;
3373   case Bytecodes::_fast_agetfield:
3374     do_oop_load(_masm, field, r0, IN_HEAP);
3375     __ verify_oop(r0);
3376     break;
3377   case Bytecodes::_fast_lgetfield:
3378     __ access_load_at(T_LONG, IN_HEAP, r0, field, noreg, noreg);
3379     break;
3380   case Bytecodes::_fast_igetfield:
3381     __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
3382     break;
3383   case Bytecodes::_fast_bgetfield:
3384     __ access_load_at(T_BYTE, IN_HEAP, r0, field, noreg, noreg);
3385     break;
3386   case Bytecodes::_fast_sgetfield:
3387     __ access_load_at(T_SHORT, IN_HEAP, r0, field, noreg, noreg);
3388     break;
3389   case Bytecodes::_fast_cgetfield:
3390     __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
3391     break;
3392   case Bytecodes::_fast_fgetfield:
3393     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3394     break;
3395   case Bytecodes::_fast_dgetfield:
3396     __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg);
3397     break;
3398   default:
3399     ShouldNotReachHere();
3400   }
3401   {
3402     Label notVolatile;
3403     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3404     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3405     __ bind(notVolatile);
3406   }
3407 }
3408 
3409 void TemplateTable::fast_xaccess(TosState state)
3410 {
3411   transition(vtos, state);
3412 
3413   // get receiver
3414   __ ldr(r0, aaddress(0));
3415   // access constant pool cache
3416   __ get_cache_and_index_at_bcp(r2, r3, 2);
3417   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3418                                   ConstantPoolCacheEntry::f2_offset())));
3419 
3420   // 8179954: We need to make sure that the code generated for
3421   // volatile accesses forms a sequentially-consistent set of
3422   // operations when combined with STLR and LDAR.  Without a leading
3423   // membar it's possible for a simple Dekker test to fail if loads
3424   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3425   // the stores in one method and we interpret the loads in another.
3426   if (! UseBarriersForVolatile) {
3427     Label notVolatile;
3428     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3429                                      ConstantPoolCacheEntry::flags_offset())));
3430     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3431     __ membar(MacroAssembler::AnyAny);
3432     __ bind(notVolatile);
3433   }
3434 
3435   // make sure exception is reported in correct bcp range (getfield is
3436   // next instruction)
3437   __ increment(rbcp);
3438   __ null_check(r0);
3439   switch (state) {
3440   case itos:
3441     __ access_load_at(T_INT, IN_HEAP, r0, Address(r0, r1, Address::lsl(0)), noreg, noreg);
3442     break;
3443   case atos:
3444     do_oop_load(_masm, Address(r0, r1, Address::lsl(0)), r0, IN_HEAP);
3445     __ verify_oop(r0);
3446     break;
3447   case ftos:
3448     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, Address(r0, r1, Address::lsl(0)), noreg, noreg);
3449     break;
3450   default:
3451     ShouldNotReachHere();
3452   }
3453 
3454   {
3455     Label notVolatile;
3456     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3457                                      ConstantPoolCacheEntry::flags_offset())));
3458     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3459     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3460     __ bind(notVolatile);
3461   }
3462 
3463   __ decrement(rbcp);
3464 }
3465 
3466 
3467 
3468 //-----------------------------------------------------------------------------
3469 // Calls
3470 
3471 void TemplateTable::count_calls(Register method, Register temp)
3472 {
3473   __ call_Unimplemented();
3474 }
3475 
3476 void TemplateTable::prepare_invoke(int byte_no,
3477                                    Register method, // linked method (or i-klass)
3478                                    Register index,  // itable index, MethodType, etc.
3479                                    Register recv,   // if caller wants to see it
3480                                    Register flags   // if caller wants to test it
3481                                    ) {
3482   // determine flags
3483   Bytecodes::Code code = bytecode();
3484   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3485   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3486   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3487   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3488   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3489   const bool load_receiver       = (recv  != noreg);
3490   const bool save_flags          = (flags != noreg);
3491   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3492   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3493   assert(flags == noreg || flags == r3, "");
3494   assert(recv  == noreg || recv  == r2, "");
3495 
3496   // setup registers & access constant pool cache
3497   if (recv  == noreg)  recv  = r2;
3498   if (flags == noreg)  flags = r3;
3499   assert_different_registers(method, index, recv, flags);
3500 
3501   // save 'interpreter return address'
3502   __ save_bcp();
3503 
3504   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3505 
3506   // maybe push appendix to arguments (just before return address)
3507   if (is_invokedynamic || is_invokehandle) {
3508     Label L_no_push;
3509     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3510     // Push the appendix as a trailing parameter.
3511     // This must be done before we get the receiver,
3512     // since the parameter_size includes it.
3513     __ push(r19);
3514     __ mov(r19, index);
3515     __ load_resolved_reference_at_index(index, r19);
3516     __ pop(r19);
3517     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3518     __ bind(L_no_push);
3519   }
3520 
3521   // load receiver if needed (note: no return address pushed yet)
3522   if (load_receiver) {
3523     __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3524     // FIXME -- is this actually correct? looks like it should be 2
3525     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3526     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3527     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3528     // __ movptr(recv, recv_addr);
3529     __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3530     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3531     __ verify_oop(recv);
3532   }
3533 
3534   // compute return type
3535   // x86 uses a shift and mask or wings it with a shift plus assert
3536   // the mask is not needed. aarch64 just uses bitfield extract
3537   __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3538   // load return address
3539   {
3540     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3541     __ mov(rscratch1, table_addr);
3542     __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3543   }
3544 }
3545 
3546 
3547 void TemplateTable::invokevirtual_helper(Register index,
3548                                          Register recv,
3549                                          Register flags)
3550 {
3551   // Uses temporary registers r0, r3
3552   assert_different_registers(index, recv, r0, r3);
3553   // Test for an invoke of a final method
3554   Label notFinal;
3555   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3556 
3557   const Register method = index;  // method must be rmethod
3558   assert(method == rmethod,
3559          "methodOop must be rmethod for interpreter calling convention");
3560 
3561   // do the call - the index is actually the method to call
3562   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3563 
3564   // It's final, need a null check here!
3565   __ null_check(recv);
3566 
3567   // profile this call
3568   __ profile_final_call(r0);
3569   __ profile_arguments_type(r0, method, r4, true);
3570 
3571   __ jump_from_interpreted(method, r0);
3572 
3573   __ bind(notFinal);
3574 
3575   // get receiver klass
3576   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3577   __ load_klass(r0, recv);
3578 
3579   // profile this call
3580   __ profile_virtual_call(r0, rlocals, r3);
3581 
3582   // get target methodOop & entry point
3583   __ lookup_virtual_method(r0, index, method);
3584   __ profile_arguments_type(r3, method, r4, true);
3585   // FIXME -- this looks completely redundant. is it?
3586   // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3587   __ jump_from_interpreted(method, r3);
3588 }
3589 
3590 void TemplateTable::invokevirtual(int byte_no)
3591 {
3592   transition(vtos, vtos);
3593   assert(byte_no == f2_byte, "use this argument");
3594 
3595   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3596 
3597   // rmethod: index (actually a Method*)
3598   // r2: receiver
3599   // r3: flags
3600 
3601   invokevirtual_helper(rmethod, r2, r3);
3602 }
3603 
3604 void TemplateTable::invokespecial(int byte_no)
3605 {
3606   transition(vtos, vtos);
3607   assert(byte_no == f1_byte, "use this argument");
3608 
3609   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3610                  r2);  // get receiver also for null check
3611   __ verify_oop(r2);
3612   __ null_check(r2);
3613   // do the call
3614   __ profile_call(r0);
3615   __ profile_arguments_type(r0, rmethod, rbcp, false);
3616   __ jump_from_interpreted(rmethod, r0);
3617 }
3618 
3619 void TemplateTable::invokestatic(int byte_no)
3620 {
3621   transition(vtos, vtos);
3622   assert(byte_no == f1_byte, "use this argument");
3623 
3624   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3625   // do the call
3626   __ profile_call(r0);
3627   __ profile_arguments_type(r0, rmethod, r4, false);
3628   __ jump_from_interpreted(rmethod, r0);
3629 }
3630 
3631 void TemplateTable::fast_invokevfinal(int byte_no)
3632 {
3633   __ call_Unimplemented();
3634 }
3635 
3636 void TemplateTable::invokeinterface(int byte_no) {
3637   transition(vtos, vtos);
3638   assert(byte_no == f1_byte, "use this argument");
3639 
3640   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 Method*
3641                  r2, r3); // recv, flags
3642 
3643   // r0: interface klass (from f1)
3644   // rmethod: method (from f2)
3645   // r2: receiver
3646   // r3: flags
3647 
3648   // First check for Object case, then private interface method,
3649   // then regular interface method.
3650 
3651   // Special case of invokeinterface called for virtual method of
3652   // java.lang.Object.  See cpCache.cpp for details.
3653   Label notObjectMethod;
3654   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notObjectMethod);
3655 
3656   invokevirtual_helper(rmethod, r2, r3);
3657   __ bind(notObjectMethod);
3658 
3659   Label no_such_interface;
3660 
3661   // Check for private method invocation - indicated by vfinal
3662   Label notVFinal;
3663   __ tbz(r3, ConstantPoolCacheEntry::is_vfinal_shift, notVFinal);
3664 
3665   // Get receiver klass into r3 - also a null check
3666   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3667   __ load_klass(r3, r2);
3668 
3669   Label subtype;
3670   __ check_klass_subtype(r3, r0, r4, subtype);
3671   // If we get here the typecheck failed
3672   __ b(no_such_interface);
3673   __ bind(subtype);
3674 
3675   __ profile_final_call(r0);
3676   __ profile_arguments_type(r0, rmethod, r4, true);
3677   __ jump_from_interpreted(rmethod, r0);
3678 
3679   __ bind(notVFinal);
3680 
3681   // Get receiver klass into r3 - also a null check
3682   __ restore_locals();
3683   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3684   __ load_klass(r3, r2);
3685 
3686   Label no_such_method;
3687 
3688   // Preserve method for throw_AbstractMethodErrorVerbose.
3689   __ mov(r16, rmethod);
3690   // Receiver subtype check against REFC.
3691   // Superklass in r0. Subklass in r3. Blows rscratch2, r13
3692   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3693                              r3, r0, noreg,
3694                              // outputs: scan temp. reg, scan temp. reg
3695                              rscratch2, r13,
3696                              no_such_interface,
3697                              /*return_method=*/false);
3698 
3699   // profile this call
3700   __ profile_virtual_call(r3, r13, r19);
3701 
3702   // Get declaring interface class from method, and itable index
3703   __ ldr(r0, Address(rmethod, Method::const_offset()));
3704   __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
3705   __ ldr(r0, Address(r0, ConstantPool::pool_holder_offset_in_bytes()));
3706   __ ldrw(rmethod, Address(rmethod, Method::itable_index_offset()));
3707   __ subw(rmethod, rmethod, Method::itable_index_max);
3708   __ negw(rmethod, rmethod);
3709 
3710   // Preserve recvKlass for throw_AbstractMethodErrorVerbose.
3711   __ mov(rlocals, r3);
3712   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3713                              rlocals, r0, rmethod,
3714                              // outputs: method, scan temp. reg
3715                              rmethod, r13,
3716                              no_such_interface);
3717 
3718   // rmethod,: methodOop to call
3719   // r2: receiver
3720   // Check for abstract method error
3721   // Note: This should be done more efficiently via a throw_abstract_method_error
3722   //       interpreter entry point and a conditional jump to it in case of a null
3723   //       method.
3724   __ cbz(rmethod, no_such_method);
3725 
3726   __ profile_arguments_type(r3, rmethod, r13, true);
3727 
3728   // do the call
3729   // r2: receiver
3730   // rmethod,: methodOop
3731   __ jump_from_interpreted(rmethod, r3);
3732   __ should_not_reach_here();
3733 
3734   // exception handling code follows...
3735   // note: must restore interpreter registers to canonical
3736   //       state for exception handling to work correctly!
3737 
3738   __ bind(no_such_method);
3739   // throw exception
3740   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3741   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3742   // Pass arguments for generating a verbose error message.
3743   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose), r3, r16);
3744   // the call_VM checks for exception, so we should never return here.
3745   __ should_not_reach_here();
3746 
3747   __ bind(no_such_interface);
3748   // throw exception
3749   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3750   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3751   // Pass arguments for generating a verbose error message.
3752   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3753                    InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose), r3, r0);
3754   // the call_VM checks for exception, so we should never return here.
3755   __ should_not_reach_here();
3756   return;
3757 }
3758 
3759 void TemplateTable::invokehandle(int byte_no) {
3760   transition(vtos, vtos);
3761   assert(byte_no == f1_byte, "use this argument");
3762 
3763   prepare_invoke(byte_no, rmethod, r0, r2);
3764   __ verify_method_ptr(r2);
3765   __ verify_oop(r2);
3766   __ null_check(r2);
3767 
3768   // FIXME: profile the LambdaForm also
3769 
3770   // r13 is safe to use here as a scratch reg because it is about to
3771   // be clobbered by jump_from_interpreted().
3772   __ profile_final_call(r13);
3773   __ profile_arguments_type(r13, rmethod, r4, true);
3774 
3775   __ jump_from_interpreted(rmethod, r0);
3776 }
3777 
3778 void TemplateTable::invokedynamic(int byte_no) {
3779   transition(vtos, vtos);
3780   assert(byte_no == f1_byte, "use this argument");
3781 
3782   prepare_invoke(byte_no, rmethod, r0);
3783 
3784   // r0: CallSite object (from cpool->resolved_references[])
3785   // rmethod: MH.linkToCallSite method (from f2)
3786 
3787   // Note:  r0_callsite is already pushed by prepare_invoke
3788 
3789   // %%% should make a type profile for any invokedynamic that takes a ref argument
3790   // profile this call
3791   __ profile_call(rbcp);
3792   __ profile_arguments_type(r3, rmethod, r13, false);
3793 
3794   __ verify_oop(r0);
3795 
3796   __ jump_from_interpreted(rmethod, r0);
3797 }
3798 
3799 
3800 //-----------------------------------------------------------------------------
3801 // Allocation
3802 
3803 void TemplateTable::_new() {
3804   transition(vtos, atos);
3805 
3806   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3807   Label slow_case;
3808   Label done;
3809   Label initialize_header;
3810   Label initialize_object; // including clearing the fields
3811 
3812   __ get_cpool_and_tags(r4, r0);
3813   // Make sure the class we're about to instantiate has been resolved.
3814   // This is done before loading InstanceKlass to be consistent with the order
3815   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3816   const int tags_offset = Array<u1>::base_offset_in_bytes();
3817   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3818   __ lea(rscratch1, Address(rscratch1, tags_offset));
3819   __ ldarb(rscratch1, rscratch1);
3820   __ cmp(rscratch1, (u1)JVM_CONSTANT_Class);
3821   __ br(Assembler::NE, slow_case);
3822 
3823   // get InstanceKlass
3824   __ load_resolved_klass_at_offset(r4, r3, r4, rscratch1);
3825 
3826   // make sure klass is initialized & doesn't have finalizer
3827   // make sure klass is fully initialized
3828   __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3829   __ cmp(rscratch1, (u1)InstanceKlass::fully_initialized);
3830   __ br(Assembler::NE, slow_case);
3831 
3832   // get instance_size in InstanceKlass (scaled to a count of bytes)
3833   __ ldrw(r3,
3834           Address(r4,
3835                   Klass::layout_helper_offset()));
3836   // test to see if it has a finalizer or is malformed in some way
3837   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3838 
3839   // Allocate the instance:
3840   //  If TLAB is enabled:
3841   //    Try to allocate in the TLAB.
3842   //    If fails, go to the slow path.
3843   //  Else If inline contiguous allocations are enabled:
3844   //    Try to allocate in eden.
3845   //    If fails due to heap end, go to slow path.
3846   //
3847   //  If TLAB is enabled OR inline contiguous is enabled:
3848   //    Initialize the allocation.
3849   //    Exit.
3850   //
3851   //  Go to slow path.
3852   const bool allow_shared_alloc =
3853     Universe::heap()->supports_inline_contig_alloc();
3854 
3855   if (UseTLAB) {
3856     __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case);
3857 
3858     if (ZeroTLAB) {
3859       // the fields have been already cleared
3860       __ b(initialize_header);
3861     } else {
3862       // initialize both the header and fields
3863       __ b(initialize_object);
3864     }
3865   } else {
3866     // Allocation in the shared Eden, if allowed.
3867     //
3868     // r3: instance size in bytes
3869     if (allow_shared_alloc) {
3870       __ eden_allocate(r0, r3, 0, r10, slow_case);
3871     }
3872   }
3873 
3874   // If UseTLAB or allow_shared_alloc are true, the object is created above and
3875   // there is an initialize need. Otherwise, skip and go to the slow path.
3876   if (UseTLAB || allow_shared_alloc) {
3877     // The object is initialized before the header.  If the object size is
3878     // zero, go directly to the header initialization.
3879     __ bind(initialize_object);
3880     __ sub(r3, r3, sizeof(oopDesc));
3881     __ cbz(r3, initialize_header);
3882 
3883     // Initialize object fields
3884     {
3885       __ add(r2, r0, sizeof(oopDesc));
3886       Label loop;
3887       __ bind(loop);
3888       __ str(zr, Address(__ post(r2, BytesPerLong)));
3889       __ sub(r3, r3, BytesPerLong);
3890       __ cbnz(r3, loop);
3891     }
3892 
3893     // initialize object header only.
3894     __ bind(initialize_header);
3895     if (UseBiasedLocking) {
3896       __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3897     } else {
3898       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3899     }
3900     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3901     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3902     __ store_klass(r0, r4);      // store klass last
3903 
3904     {
3905       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3906       // Trigger dtrace event for fastpath
3907       __ push(atos); // save the return value
3908       __ call_VM_leaf(
3909            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3910       __ pop(atos); // restore the return value
3911 
3912     }
3913     __ b(done);
3914   }
3915 
3916   // slow case
3917   __ bind(slow_case);
3918   __ get_constant_pool(c_rarg1);
3919   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3920   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3921   __ verify_oop(r0);
3922 
3923   // continue
3924   __ bind(done);
3925   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3926   __ membar(Assembler::StoreStore);
3927 }
3928 
3929 void TemplateTable::defaultvalue() {
3930   transition(vtos, atos);
3931   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3932   __ get_constant_pool(c_rarg1);
3933   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::defaultvalue),
3934           c_rarg1, c_rarg2);
3935   __ verify_oop(r0);
3936   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3937   __ membar(Assembler::StoreStore);
3938 }
3939 
3940 void TemplateTable::withfield() {
3941   transition(vtos, atos);
3942   resolve_cache_and_index(f2_byte, c_rarg1 /*cache*/, c_rarg2 /*index*/, sizeof(u2));
3943 
3944   // n.b. unlike x86 cache is now rcpool plus the indexed offset
3945   // so using rcpool to meet shared code expectations
3946  
3947   call_VM(r1, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), rcpool);
3948   __ verify_oop(r1);
3949   __ add(esp, esp, r0);
3950   __ mov(r0, r1);
3951 }
3952 
3953 void TemplateTable::newarray() {
3954   transition(itos, atos);
3955   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3956   __ mov(c_rarg2, r0);
3957   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3958           c_rarg1, c_rarg2);
3959   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3960   __ membar(Assembler::StoreStore);
3961 }
3962 
3963 void TemplateTable::anewarray() {
3964   transition(itos, atos);
3965   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3966   __ get_constant_pool(c_rarg1);
3967   __ mov(c_rarg3, r0);
3968   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3969           c_rarg1, c_rarg2, c_rarg3);
3970   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3971   __ membar(Assembler::StoreStore);
3972 }
3973 
3974 void TemplateTable::arraylength() {
3975   transition(atos, itos);
3976   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3977   __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3978 }
3979 
3980 void TemplateTable::checkcast()
3981 {
3982   transition(atos, atos);
3983   Label done, is_null, ok_is_subtype, quicked, resolved;
3984   __ cbz(r0, is_null);
3985 
3986   // Get cpool & tags index
3987   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3988   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3989   // See if bytecode has already been quicked
3990   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3991   __ lea(r1, Address(rscratch1, r19));
3992   __ ldarb(r1, r1);
3993   __ cmp(r1, (u1)JVM_CONSTANT_Class);
3994   __ br(Assembler::EQ, quicked);
3995 
3996   __ push(atos); // save receiver for result, and for GC
3997   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3998   // vm_result_2 has metadata result
3999   __ get_vm_result_2(r0, rthread);
4000   __ pop(r3); // restore receiver
4001   __ b(resolved);
4002 
4003   // Get superklass in r0 and subklass in r3
4004   __ bind(quicked);
4005   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
4006   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1); // r0 = klass
4007 
4008   __ bind(resolved);
4009   __ load_klass(r19, r3);
4010 
4011   // Generate subtype check.  Blows r2, r5.  Object in r3.
4012   // Superklass in r0.  Subklass in r19.
4013   __ gen_subtype_check(r19, ok_is_subtype);
4014 
4015   // Come here on failure
4016   __ push(r3);
4017   // object is at TOS
4018   __ b(Interpreter::_throw_ClassCastException_entry);
4019 
4020   // Come here on success
4021   __ bind(ok_is_subtype);
4022   __ mov(r0, r3); // Restore object in r3
4023 
4024   __ b(done);
4025   __ bind(is_null);
4026 
4027   // Collect counts on whether this test sees NULLs a lot or not.
4028   if (ProfileInterpreter) {
4029     __ profile_null_seen(r2);
4030   }
4031 
4032   if (EnableValhalla) {
4033     // Get cpool & tags index
4034     __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
4035     __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
4036      // See if bytecode has already been quicked
4037     __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
4038     __ lea(r1, Address(rscratch1, r19));
4039     __ ldarb(r1, r1); 
4040     // See if CP entry is a Q-descriptor
4041     __ andr (r1, r1, JVM_CONSTANT_QDescBit);
4042     __ cmp(r1, (u1) JVM_CONSTANT_QDescBit);
4043     __ br(Assembler::NE, done);
4044     __ b(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
4045   }
4046 
4047   __ bind(done);
4048 }
4049 
4050 void TemplateTable::instanceof() {
4051   transition(atos, itos);
4052   Label done, is_null, ok_is_subtype, quicked, resolved;
4053   __ cbz(r0, is_null);
4054 
4055   // Get cpool & tags index
4056   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
4057   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
4058   // See if bytecode has already been quicked
4059   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
4060   __ lea(r1, Address(rscratch1, r19));
4061   __ ldarb(r1, r1);
4062   __ cmp(r1, (u1)JVM_CONSTANT_Class);
4063   __ br(Assembler::EQ, quicked);
4064 
4065   __ push(atos); // save receiver for result, and for GC
4066   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4067   // vm_result_2 has metadata result
4068   __ get_vm_result_2(r0, rthread);
4069   __ pop(r3); // restore receiver
4070   __ verify_oop(r3);
4071   __ load_klass(r3, r3);
4072   __ b(resolved);
4073 
4074   // Get superklass in r0 and subklass in r3
4075   __ bind(quicked);
4076   __ load_klass(r3, r0);
4077   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1);
4078 
4079   __ bind(resolved);
4080 
4081   // Generate subtype check.  Blows r2, r5
4082   // Superklass in r0.  Subklass in r3.
4083   __ gen_subtype_check(r3, ok_is_subtype);
4084 
4085   // Come here on failure
4086   __ mov(r0, 0);
4087   __ b(done);
4088   // Come here on success
4089   __ bind(ok_is_subtype);
4090   __ mov(r0, 1);
4091 
4092   // Collect counts on whether this test sees NULLs a lot or not.
4093   if (ProfileInterpreter) {
4094     __ b(done);
4095     __ bind(is_null);
4096     __ profile_null_seen(r2);
4097   } else {
4098     __ bind(is_null);   // same as 'done'
4099   }
4100   __ bind(done);
4101   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
4102   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
4103 }
4104 
4105 //-----------------------------------------------------------------------------
4106 // Breakpoints
4107 void TemplateTable::_breakpoint() {
4108   // Note: We get here even if we are single stepping..
4109   // jbug inists on setting breakpoints at every bytecode
4110   // even if we are in single step mode.
4111 
4112   transition(vtos, vtos);
4113 
4114   // get the unpatched byte code
4115   __ get_method(c_rarg1);
4116   __ call_VM(noreg,
4117              CAST_FROM_FN_PTR(address,
4118                               InterpreterRuntime::get_original_bytecode_at),
4119              c_rarg1, rbcp);
4120   __ mov(r19, r0);
4121 
4122   // post the breakpoint event
4123   __ call_VM(noreg,
4124              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
4125              rmethod, rbcp);
4126 
4127   // complete the execution of original bytecode
4128   __ mov(rscratch1, r19);
4129   __ dispatch_only_normal(vtos);
4130 }
4131 
4132 //-----------------------------------------------------------------------------
4133 // Exceptions
4134 
4135 void TemplateTable::athrow() {
4136   transition(atos, vtos);
4137   __ null_check(r0);
4138   __ b(Interpreter::throw_exception_entry());
4139 }
4140 
4141 //-----------------------------------------------------------------------------
4142 // Synchronization
4143 //
4144 // Note: monitorenter & exit are symmetric routines; which is reflected
4145 //       in the assembly code structure as well
4146 //
4147 // Stack layout:
4148 //
4149 // [expressions  ] <--- esp               = expression stack top
4150 // ..
4151 // [expressions  ]
4152 // [monitor entry] <--- monitor block top = expression stack bot
4153 // ..
4154 // [monitor entry]
4155 // [frame data   ] <--- monitor block bot
4156 // ...
4157 // [saved rbp    ] <--- rbp
4158 void TemplateTable::monitorenter()
4159 {
4160   transition(atos, vtos);
4161 
4162   // check for NULL object
4163   __ null_check(r0);
4164 
4165   __ resolve(IS_NOT_NULL, r0);
4166 
4167   const Address monitor_block_top(
4168         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4169   const Address monitor_block_bot(
4170         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4171   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4172 
4173   Label allocated;
4174 
4175   // initialize entry pointer
4176   __ mov(c_rarg1, zr); // points to free slot or NULL
4177 
4178   // find a free slot in the monitor block (result in c_rarg1)
4179   {
4180     Label entry, loop, exit;
4181     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
4182                                         // starting with top-most entry
4183     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4184 
4185     __ b(entry);
4186 
4187     __ bind(loop);
4188     // check if current entry is used
4189     // if not used then remember entry in c_rarg1
4190     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
4191     __ cmp(zr, rscratch1);
4192     __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
4193     // check if current entry is for same object
4194     __ cmp(r0, rscratch1);
4195     // if same object then stop searching
4196     __ br(Assembler::EQ, exit);
4197     // otherwise advance to next entry
4198     __ add(c_rarg3, c_rarg3, entry_size);
4199     __ bind(entry);
4200     // check if bottom reached
4201     __ cmp(c_rarg3, c_rarg2);
4202     // if not at bottom then check this entry
4203     __ br(Assembler::NE, loop);
4204     __ bind(exit);
4205   }
4206 
4207   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
4208                             // if found, continue with that on
4209 
4210   // allocate one if there's no free slot
4211   {
4212     Label entry, loop;
4213     // 1. compute new pointers            // rsp: old expression stack top
4214     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
4215     __ sub(esp, esp, entry_size);         // move expression stack top
4216     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
4217     __ mov(c_rarg3, esp);                 // set start value for copy loop
4218     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
4219 
4220     __ sub(sp, sp, entry_size);           // make room for the monitor
4221 
4222     __ b(entry);
4223     // 2. move expression stack contents
4224     __ bind(loop);
4225     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
4226                                                    // word from old location
4227     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
4228     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
4229     __ bind(entry);
4230     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
4231     __ br(Assembler::NE, loop);      // if not at bottom then
4232                                      // copy next word
4233   }
4234 
4235   // call run-time routine
4236   // c_rarg1: points to monitor entry
4237   __ bind(allocated);
4238 
4239   // Increment bcp to point to the next bytecode, so exception
4240   // handling for async. exceptions work correctly.
4241   // The object has already been poped from the stack, so the
4242   // expression stack looks correct.
4243   __ increment(rbcp);
4244 
4245   // store object
4246   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
4247   __ lock_object(c_rarg1);
4248 
4249   // check to make sure this monitor doesn't cause stack overflow after locking
4250   __ save_bcp();  // in case of exception
4251   __ generate_stack_overflow_check(0);
4252 
4253   // The bcp has already been incremented. Just need to dispatch to
4254   // next instruction.
4255   __ dispatch_next(vtos);
4256 }
4257 
4258 
4259 void TemplateTable::monitorexit()
4260 {
4261   transition(atos, vtos);
4262 
4263   // check for NULL object
4264   __ null_check(r0);
4265 
4266   __ resolve(IS_NOT_NULL, r0);
4267 
4268   const Address monitor_block_top(
4269         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4270   const Address monitor_block_bot(
4271         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4272   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4273 
4274   Label found;
4275 
4276   // find matching slot
4277   {
4278     Label entry, loop;
4279     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
4280                                         // starting with top-most entry
4281     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4282                                         // of monitor block
4283     __ b(entry);
4284 
4285     __ bind(loop);
4286     // check if current entry is for same object
4287     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
4288     __ cmp(r0, rscratch1);
4289     // if same object then stop searching
4290     __ br(Assembler::EQ, found);
4291     // otherwise advance to next entry
4292     __ add(c_rarg1, c_rarg1, entry_size);
4293     __ bind(entry);
4294     // check if bottom reached
4295     __ cmp(c_rarg1, c_rarg2);
4296     // if not at bottom then check this entry
4297     __ br(Assembler::NE, loop);
4298   }
4299 
4300   // error handling. Unlocking was not block-structured
4301   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4302                    InterpreterRuntime::throw_illegal_monitor_state_exception));
4303   __ should_not_reach_here();
4304 
4305   // call run-time routine
4306   __ bind(found);
4307   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
4308   __ unlock_object(c_rarg1);
4309   __ pop_ptr(r0); // discard object
4310 }
4311 
4312 
4313 // Wide instructions
4314 void TemplateTable::wide()
4315 {
4316   __ load_unsigned_byte(r19, at_bcp(1));
4317   __ mov(rscratch1, (address)Interpreter::_wentry_point);
4318   __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
4319   __ br(rscratch1);
4320 }
4321 
4322 
4323 // Multi arrays
4324 void TemplateTable::multianewarray() {
4325   transition(vtos, atos);
4326   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
4327   // last dim is on top of stack; we want address of first one:
4328   // first_addr = last_addr + (ndims - 1) * wordSize
4329   __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
4330   __ sub(c_rarg1, c_rarg1, wordSize);
4331   call_VM(r0,
4332           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
4333           c_rarg1);
4334   __ load_unsigned_byte(r1, at_bcp(3));
4335   __ lea(esp, Address(esp, r1, Address::uxtw(3)));
4336 }