1 /*
   2  * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "interpreter/interpreterRuntime.hpp"
  30 #include "interpreter/interp_masm.hpp"
  31 #include "interpreter/templateTable.hpp"
  32 #include "memory/universe.inline.hpp"
  33 #include "oops/methodData.hpp"
  34 #include "oops/method.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/synchronizer.hpp"
  41 
  42 #define __ _masm->
  43 
  44 // Platform-dependent initialization
  45 
  46 void TemplateTable::pd_initialize() {
  47   // No aarch64 specific initialization
  48 }
  49 
  50 // Address computation: local variables
  51 
  52 static inline Address iaddress(int n) {
  53   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  54 }
  55 
  56 static inline Address laddress(int n) {
  57   return iaddress(n + 1);
  58 }
  59 
  60 static inline Address faddress(int n) {
  61   return iaddress(n);
  62 }
  63 
  64 static inline Address daddress(int n) {
  65   return laddress(n);
  66 }
  67 
  68 static inline Address aaddress(int n) {
  69   return iaddress(n);
  70 }
  71 
  72 static inline Address iaddress(Register r) {
  73   return Address(rlocals, r, Address::lsl(3));
  74 }
  75 
  76 static inline Address laddress(Register r, Register scratch,
  77                                InterpreterMacroAssembler* _masm) {
  78   __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
  79   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  80 }
  81 
  82 static inline Address faddress(Register r) {
  83   return iaddress(r);
  84 }
  85 
  86 static inline Address daddress(Register r, Register scratch,
  87                                InterpreterMacroAssembler* _masm) {
  88   return laddress(r, scratch, _masm);
  89 }
  90 
  91 static inline Address aaddress(Register r) {
  92   return iaddress(r);
  93 }
  94 
  95 static inline Address at_rsp() {
  96   return Address(esp, 0);
  97 }
  98 
  99 // At top of Java expression stack which may be different than esp().  It
 100 // isn't for category 1 objects.
 101 static inline Address at_tos   () {
 102   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 103 }
 104 
 105 static inline Address at_tos_p1() {
 106   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 107 }
 108 
 109 static inline Address at_tos_p2() {
 110   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 111 }
 112 
 113 static inline Address at_tos_p3() {
 114   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 115 }
 116 
 117 static inline Address at_tos_p4() {
 118   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 119 }
 120 
 121 static inline Address at_tos_p5() {
 122   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 123 }
 124 
 125 // Condition conversion
 126 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 127   switch (cc) {
 128   case TemplateTable::equal        : return Assembler::NE;
 129   case TemplateTable::not_equal    : return Assembler::EQ;
 130   case TemplateTable::less         : return Assembler::GE;
 131   case TemplateTable::less_equal   : return Assembler::GT;
 132   case TemplateTable::greater      : return Assembler::LE;
 133   case TemplateTable::greater_equal: return Assembler::LT;
 134   }
 135   ShouldNotReachHere();
 136   return Assembler::EQ;
 137 }
 138 
 139 
 140 // Miscelaneous helper routines
 141 // Store an oop (or NULL) at the Address described by obj.
 142 // If val == noreg this means store a NULL
 143 static void do_oop_store(InterpreterMacroAssembler* _masm,
 144                          Address obj,
 145                          Register val,
 146                          BarrierSet::Name barrier,
 147                          bool precise) {
 148   assert(val == noreg || val == r0, "parameter is just for looks");
 149   switch (barrier) {
 150 #if INCLUDE_ALL_GCS
 151     case BarrierSet::G1SATBCTLogging:
 152       {
 153         // flatten object address if needed
 154         if (obj.index() == noreg && obj.offset() == 0) {
 155           if (obj.base() != r3) {
 156             __ mov(r3, obj.base());
 157           }
 158         } else {
 159           __ lea(r3, obj);
 160         }
 161         __ g1_write_barrier_pre(r3 /* obj */,
 162                                 r1 /* pre_val */,
 163                                 rthread /* thread */,
 164                                 r10  /* tmp */,
 165                                 val != noreg /* tosca_live */,
 166                                 false /* expand_call */);
 167         if (val == noreg) {
 168           __ store_heap_oop_null(Address(r3, 0));
 169         } else {
 170           // G1 barrier needs uncompressed oop for region cross check.
 171           Register new_val = val;
 172           if (UseCompressedOops) {
 173             new_val = rscratch1;
 174             __ mov(new_val, val);
 175           }
 176           __ store_heap_oop(Address(r3, 0), val);
 177           __ g1_write_barrier_post(r3 /* store_adr */,
 178                                    new_val /* new_val */,
 179                                    rthread /* thread */,
 180                                    r10 /* tmp */,
 181                                    r1 /* tmp2 */);
 182         }
 183 
 184       }
 185       break;
 186 #endif // INCLUDE_ALL_GCS
 187     case BarrierSet::CardTableForRS:
 188     case BarrierSet::CardTableExtension:
 189       {
 190         if (val == noreg) {
 191           __ store_heap_oop_null(obj);
 192         } else {
 193           __ store_heap_oop(obj, val);
 194           // flatten object address if needed
 195           if (!precise || (obj.index() == noreg && obj.offset() == 0)) {
 196             __ store_check(obj.base());
 197           } else {
 198             __ lea(r3, obj);
 199             __ store_check(r3);
 200           }
 201         }
 202       }
 203       break;
 204     case BarrierSet::ModRef:
 205       if (val == noreg) {
 206         __ store_heap_oop_null(obj);
 207       } else {
 208         __ store_heap_oop(obj, val);
 209       }
 210       break;
 211     default      :
 212       ShouldNotReachHere();
 213 
 214   }
 215 }
 216 
 217 Address TemplateTable::at_bcp(int offset) {
 218   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 219   return Address(rbcp, offset);
 220 }
 221 
 222 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 223                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 224                                    int byte_no)
 225 {
 226   if (!RewriteBytecodes)  return;
 227   Label L_patch_done;
 228 
 229   switch (bc) {
 230   case Bytecodes::_fast_aputfield:
 231   case Bytecodes::_fast_bputfield:
 232   case Bytecodes::_fast_zputfield:
 233   case Bytecodes::_fast_cputfield:
 234   case Bytecodes::_fast_dputfield:
 235   case Bytecodes::_fast_fputfield:
 236   case Bytecodes::_fast_iputfield:
 237   case Bytecodes::_fast_lputfield:
 238   case Bytecodes::_fast_sputfield:
 239     {
 240       // We skip bytecode quickening for putfield instructions when
 241       // the put_code written to the constant pool cache is zero.
 242       // This is required so that every execution of this instruction
 243       // calls out to InterpreterRuntime::resolve_get_put to do
 244       // additional, required work.
 245       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 246       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 247       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 248       __ movw(bc_reg, bc);
 249       __ cmpw(temp_reg, (unsigned) 0);
 250       __ br(Assembler::EQ, L_patch_done);  // don't patch
 251     }
 252     break;
 253   default:
 254     assert(byte_no == -1, "sanity");
 255     // the pair bytecodes have already done the load.
 256     if (load_bc_into_bc_reg) {
 257       __ movw(bc_reg, bc);
 258     }
 259   }
 260 
 261   if (JvmtiExport::can_post_breakpoint()) {
 262     Label L_fast_patch;
 263     // if a breakpoint is present we can't rewrite the stream directly
 264     __ load_unsigned_byte(temp_reg, at_bcp(0));
 265     __ cmpw(temp_reg, Bytecodes::_breakpoint);
 266     __ br(Assembler::NE, L_fast_patch);
 267     // Let breakpoint table handling rewrite to quicker bytecode
 268     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 269     __ b(L_patch_done);
 270     __ bind(L_fast_patch);
 271   }
 272 
 273 #ifdef ASSERT
 274   Label L_okay;
 275   __ load_unsigned_byte(temp_reg, at_bcp(0));
 276   __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
 277   __ br(Assembler::EQ, L_okay);
 278   __ cmpw(temp_reg, bc_reg);
 279   __ br(Assembler::EQ, L_okay);
 280   __ stop("patching the wrong bytecode");
 281   __ bind(L_okay);
 282 #endif
 283 
 284   // patch bytecode
 285   __ strb(bc_reg, at_bcp(0));
 286   __ bind(L_patch_done);
 287 }
 288 
 289 
 290 // Individual instructions
 291 
 292 void TemplateTable::nop() {
 293   transition(vtos, vtos);
 294   // nothing to do
 295 }
 296 
 297 void TemplateTable::shouldnotreachhere() {
 298   transition(vtos, vtos);
 299   __ stop("shouldnotreachhere bytecode");
 300 }
 301 
 302 void TemplateTable::aconst_null()
 303 {
 304   transition(vtos, atos);
 305   __ mov(r0, 0);
 306 }
 307 
 308 void TemplateTable::iconst(int value)
 309 {
 310   transition(vtos, itos);
 311   __ mov(r0, value);
 312 }
 313 
 314 void TemplateTable::lconst(int value)
 315 {
 316   __ mov(r0, value);
 317 }
 318 
 319 void TemplateTable::fconst(int value)
 320 {
 321   transition(vtos, ftos);
 322   switch (value) {
 323   case 0:
 324     __ fmovs(v0, zr);
 325     break;
 326   case 1:
 327     __ fmovs(v0, 1.0);
 328     break;
 329   case 2:
 330     __ fmovs(v0, 2.0);
 331     break;
 332   default:
 333     ShouldNotReachHere();
 334     break;
 335   }
 336 }
 337 
 338 void TemplateTable::dconst(int value)
 339 {
 340   transition(vtos, dtos);
 341   switch (value) {
 342   case 0:
 343     __ fmovd(v0, zr);
 344     break;
 345   case 1:
 346     __ fmovd(v0, 1.0);
 347     break;
 348   case 2:
 349     __ fmovd(v0, 2.0);
 350     break;
 351   default:
 352     ShouldNotReachHere();
 353     break;
 354   }
 355 }
 356 
 357 void TemplateTable::bipush()
 358 {
 359   transition(vtos, itos);
 360   __ load_signed_byte32(r0, at_bcp(1));
 361 }
 362 
 363 void TemplateTable::sipush()
 364 {
 365   transition(vtos, itos);
 366   __ load_unsigned_short(r0, at_bcp(1));
 367   __ revw(r0, r0);
 368   __ asrw(r0, r0, 16);
 369 }
 370 
 371 void TemplateTable::ldc(bool wide)
 372 {
 373   transition(vtos, vtos);
 374   Label call_ldc, notFloat, notClass, Done;
 375 
 376   if (wide) {
 377     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 378   } else {
 379     __ load_unsigned_byte(r1, at_bcp(1));
 380   }
 381   __ get_cpool_and_tags(r2, r0);
 382 
 383   const int base_offset = ConstantPool::header_size() * wordSize;
 384   const int tags_offset = Array<u1>::base_offset_in_bytes();
 385 
 386   // get type
 387   __ add(r3, r1, tags_offset);
 388   __ lea(r3, Address(r0, r3));
 389   __ ldarb(r3, r3);
 390 
 391   // unresolved class - get the resolved class
 392   __ cmp(r3, JVM_CONSTANT_UnresolvedClass);
 393   __ br(Assembler::EQ, call_ldc);
 394 
 395   // unresolved class in error state - call into runtime to throw the error
 396   // from the first resolution attempt
 397   __ cmp(r3, JVM_CONSTANT_UnresolvedClassInError);
 398   __ br(Assembler::EQ, call_ldc);
 399 
 400   // resolved class - need to call vm to get java mirror of the class
 401   __ cmp(r3, JVM_CONSTANT_Class);
 402   __ br(Assembler::NE, notClass);
 403 
 404   __ bind(call_ldc);
 405   __ mov(c_rarg1, wide);
 406   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 407   __ push_ptr(r0);
 408   __ verify_oop(r0);
 409   __ b(Done);
 410 
 411   __ bind(notClass);
 412   __ cmp(r3, JVM_CONSTANT_Float);
 413   __ br(Assembler::NE, notFloat);
 414   // ftos
 415   __ adds(r1, r2, r1, Assembler::LSL, 3);
 416   __ ldrs(v0, Address(r1, base_offset));
 417   __ push_f();
 418   __ b(Done);
 419 
 420   __ bind(notFloat);
 421 #ifdef ASSERT
 422   {
 423     Label L;
 424     __ cmp(r3, JVM_CONSTANT_Integer);
 425     __ br(Assembler::EQ, L);
 426     // String and Object are rewritten to fast_aldc
 427     __ stop("unexpected tag type in ldc");
 428     __ bind(L);
 429   }
 430 #endif
 431   // itos JVM_CONSTANT_Integer only
 432   __ adds(r1, r2, r1, Assembler::LSL, 3);
 433   __ ldrw(r0, Address(r1, base_offset));
 434   __ push_i(r0);
 435   __ bind(Done);
 436 }
 437 
 438 // Fast path for caching oop constants.
 439 void TemplateTable::fast_aldc(bool wide)
 440 {
 441   transition(vtos, atos);
 442 
 443   Register result = r0;
 444   Register tmp = r1;
 445   int index_size = wide ? sizeof(u2) : sizeof(u1);
 446 
 447   Label resolved;
 448 
 449   // We are resolved if the resolved reference cache entry contains a
 450   // non-null object (String, MethodType, etc.)
 451   assert_different_registers(result, tmp);
 452   __ get_cache_index_at_bcp(tmp, 1, index_size);
 453   __ load_resolved_reference_at_index(result, tmp);
 454   __ cbnz(result, resolved);
 455 
 456   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 457 
 458   // first time invocation - must resolve first
 459   __ mov(tmp, (int)bytecode());
 460   __ call_VM(result, entry, tmp);
 461 
 462   __ bind(resolved);
 463 
 464   if (VerifyOops) {
 465     __ verify_oop(result);
 466   }
 467 }
 468 
 469 void TemplateTable::ldc2_w()
 470 {
 471   transition(vtos, vtos);
 472   Label Long, Done;
 473   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 474 
 475   __ get_cpool_and_tags(r1, r2);
 476   const int base_offset = ConstantPool::header_size() * wordSize;
 477   const int tags_offset = Array<u1>::base_offset_in_bytes();
 478 
 479   // get type
 480   __ lea(r2, Address(r2, r0, Address::lsl(0)));
 481   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 482   __ cmpw(r2, (int)JVM_CONSTANT_Double);
 483   __ br(Assembler::NE, Long);
 484   // dtos
 485   __ lea (r2, Address(r1, r0, Address::lsl(3)));
 486   __ ldrd(v0, Address(r2, base_offset));
 487   __ push_d();
 488   __ b(Done);
 489 
 490   __ bind(Long);
 491   // ltos
 492   __ lea(r0, Address(r1, r0, Address::lsl(3)));
 493   __ ldr(r0, Address(r0, base_offset));
 494   __ push_l();
 495 
 496   __ bind(Done);
 497 }
 498 
 499 void TemplateTable::locals_index(Register reg, int offset)
 500 {
 501   __ ldrb(reg, at_bcp(offset));
 502   __ neg(reg, reg);
 503 }
 504 
 505 void TemplateTable::iload() {
 506   iload_internal();
 507 }
 508 
 509 void TemplateTable::nofast_iload() {
 510   iload_internal(may_not_rewrite);
 511 }
 512 
 513 void TemplateTable::iload_internal(RewriteControl rc) {
 514   transition(vtos, itos);
 515   if (RewriteFrequentPairs && rc == may_rewrite) {
 516     Label rewrite, done;
 517     Register bc = r4;
 518 
 519     // get next bytecode
 520     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 521 
 522     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 523     // last two iloads in a pair.  Comparing against fast_iload means that
 524     // the next bytecode is neither an iload or a caload, and therefore
 525     // an iload pair.
 526     __ cmpw(r1, Bytecodes::_iload);
 527     __ br(Assembler::EQ, done);
 528 
 529     // if _fast_iload rewrite to _fast_iload2
 530     __ cmpw(r1, Bytecodes::_fast_iload);
 531     __ movw(bc, Bytecodes::_fast_iload2);
 532     __ br(Assembler::EQ, rewrite);
 533 
 534     // if _caload rewrite to _fast_icaload
 535     __ cmpw(r1, Bytecodes::_caload);
 536     __ movw(bc, Bytecodes::_fast_icaload);
 537     __ br(Assembler::EQ, rewrite);
 538 
 539     // else rewrite to _fast_iload
 540     __ movw(bc, Bytecodes::_fast_iload);
 541 
 542     // rewrite
 543     // bc: new bytecode
 544     __ bind(rewrite);
 545     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 546     __ bind(done);
 547 
 548   }
 549 
 550   // do iload, get the local value into tos
 551   locals_index(r1);
 552   __ ldr(r0, iaddress(r1));
 553 
 554 }
 555 
 556 void TemplateTable::fast_iload2()
 557 {
 558   transition(vtos, itos);
 559   locals_index(r1);
 560   __ ldr(r0, iaddress(r1));
 561   __ push(itos);
 562   locals_index(r1, 3);
 563   __ ldr(r0, iaddress(r1));
 564 }
 565 
 566 void TemplateTable::fast_iload()
 567 {
 568   transition(vtos, itos);
 569   locals_index(r1);
 570   __ ldr(r0, iaddress(r1));
 571 }
 572 
 573 void TemplateTable::lload()
 574 {
 575   transition(vtos, ltos);
 576   __ ldrb(r1, at_bcp(1));
 577   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 578   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 579 }
 580 
 581 void TemplateTable::fload()
 582 {
 583   transition(vtos, ftos);
 584   locals_index(r1);
 585   // n.b. we use ldrd here because this is a 64 bit slot
 586   // this is comparable to the iload case
 587   __ ldrd(v0, faddress(r1));
 588 }
 589 
 590 void TemplateTable::dload()
 591 {
 592   transition(vtos, dtos);
 593   __ ldrb(r1, at_bcp(1));
 594   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 595   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 596 }
 597 
 598 void TemplateTable::aload()
 599 {
 600   transition(vtos, atos);
 601   locals_index(r1);
 602   __ ldr(r0, iaddress(r1));
 603 }
 604 
 605 void TemplateTable::locals_index_wide(Register reg) {
 606   __ ldrh(reg, at_bcp(2));
 607   __ rev16w(reg, reg);
 608   __ neg(reg, reg);
 609 }
 610 
 611 void TemplateTable::wide_iload() {
 612   transition(vtos, itos);
 613   locals_index_wide(r1);
 614   __ ldr(r0, iaddress(r1));
 615 }
 616 
 617 void TemplateTable::wide_lload()
 618 {
 619   transition(vtos, ltos);
 620   __ ldrh(r1, at_bcp(2));
 621   __ rev16w(r1, r1);
 622   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 623   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 624 }
 625 
 626 void TemplateTable::wide_fload()
 627 {
 628   transition(vtos, ftos);
 629   locals_index_wide(r1);
 630   // n.b. we use ldrd here because this is a 64 bit slot
 631   // this is comparable to the iload case
 632   __ ldrd(v0, faddress(r1));
 633 }
 634 
 635 void TemplateTable::wide_dload()
 636 {
 637   transition(vtos, dtos);
 638   __ ldrh(r1, at_bcp(2));
 639   __ rev16w(r1, r1);
 640   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 641   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 642 }
 643 
 644 void TemplateTable::wide_aload()
 645 {
 646   transition(vtos, atos);
 647   locals_index_wide(r1);
 648   __ ldr(r0, aaddress(r1));
 649 }
 650 
 651 void TemplateTable::index_check(Register array, Register index)
 652 {
 653   // destroys r1, rscratch1
 654   // check array
 655   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 656   // sign extend index for use by indexed load
 657   // __ movl2ptr(index, index);
 658   // check index
 659   Register length = rscratch1;
 660   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 661   __ cmpw(index, length);
 662   if (index != r1) {
 663     // ??? convention: move aberrant index into r1 for exception message
 664     assert(r1 != array, "different registers");
 665     __ mov(r1, index);
 666   }
 667   Label ok;
 668   __ br(Assembler::LO, ok);
 669   __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 670   __ br(rscratch1);
 671   __ bind(ok);
 672 }
 673 
 674 void TemplateTable::iaload()
 675 {
 676   transition(itos, itos);
 677   __ mov(r1, r0);
 678   __ pop_ptr(r0);
 679   // r0: array
 680   // r1: index
 681   index_check(r0, r1); // leaves index in r1, kills rscratch1
 682   __ lea(r1, Address(r0, r1, Address::uxtw(2)));
 683   __ ldrw(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_INT)));
 684 }
 685 
 686 void TemplateTable::laload()
 687 {
 688   transition(itos, ltos);
 689   __ mov(r1, r0);
 690   __ pop_ptr(r0);
 691   // r0: array
 692   // r1: index
 693   index_check(r0, r1); // leaves index in r1, kills rscratch1
 694   __ lea(r1, Address(r0, r1, Address::uxtw(3)));
 695   __ ldr(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_LONG)));
 696 }
 697 
 698 void TemplateTable::faload()
 699 {
 700   transition(itos, ftos);
 701   __ mov(r1, r0);
 702   __ pop_ptr(r0);
 703   // r0: array
 704   // r1: index
 705   index_check(r0, r1); // leaves index in r1, kills rscratch1
 706   __ lea(r1,  Address(r0, r1, Address::uxtw(2)));
 707   __ ldrs(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
 708 }
 709 
 710 void TemplateTable::daload()
 711 {
 712   transition(itos, dtos);
 713   __ mov(r1, r0);
 714   __ pop_ptr(r0);
 715   // r0: array
 716   // r1: index
 717   index_check(r0, r1); // leaves index in r1, kills rscratch1
 718   __ lea(r1,  Address(r0, r1, Address::uxtw(3)));
 719   __ ldrd(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
 720 }
 721 
 722 void TemplateTable::aaload()
 723 {
 724   transition(itos, atos);
 725   __ mov(r1, r0);
 726   __ pop_ptr(r0);
 727   // r0: array
 728   // r1: index
 729   index_check(r0, r1); // leaves index in r1, kills rscratch1
 730   int s = (UseCompressedOops ? 2 : 3);
 731   __ lea(r1, Address(r0, r1, Address::uxtw(s)));
 732   __ load_heap_oop(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 733 }
 734 
 735 void TemplateTable::baload()
 736 {
 737   transition(itos, itos);
 738   __ mov(r1, r0);
 739   __ pop_ptr(r0);
 740   // r0: array
 741   // r1: index
 742   index_check(r0, r1); // leaves index in r1, kills rscratch1
 743   __ lea(r1,  Address(r0, r1, Address::uxtw(0)));
 744   __ load_signed_byte(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 745 }
 746 
 747 void TemplateTable::caload()
 748 {
 749   transition(itos, itos);
 750   __ mov(r1, r0);
 751   __ pop_ptr(r0);
 752   // r0: array
 753   // r1: index
 754   index_check(r0, r1); // leaves index in r1, kills rscratch1
 755   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 756   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 757 }
 758 
 759 // iload followed by caload frequent pair
 760 void TemplateTable::fast_icaload()
 761 {
 762   transition(vtos, itos);
 763   // load index out of locals
 764   locals_index(r2);
 765   __ ldr(r1, iaddress(r2));
 766 
 767   __ pop_ptr(r0);
 768 
 769   // r0: array
 770   // r1: index
 771   index_check(r0, r1); // leaves index in r1, kills rscratch1
 772   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 773   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 774 }
 775 
 776 void TemplateTable::saload()
 777 {
 778   transition(itos, itos);
 779   __ mov(r1, r0);
 780   __ pop_ptr(r0);
 781   // r0: array
 782   // r1: index
 783   index_check(r0, r1); // leaves index in r1, kills rscratch1
 784   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 785   __ load_signed_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 786 }
 787 
 788 void TemplateTable::iload(int n)
 789 {
 790   transition(vtos, itos);
 791   __ ldr(r0, iaddress(n));
 792 }
 793 
 794 void TemplateTable::lload(int n)
 795 {
 796   transition(vtos, ltos);
 797   __ ldr(r0, laddress(n));
 798 }
 799 
 800 void TemplateTable::fload(int n)
 801 {
 802   transition(vtos, ftos);
 803   __ ldrs(v0, faddress(n));
 804 }
 805 
 806 void TemplateTable::dload(int n)
 807 {
 808   transition(vtos, dtos);
 809   __ ldrd(v0, daddress(n));
 810 }
 811 
 812 void TemplateTable::aload(int n)
 813 {
 814   transition(vtos, atos);
 815   __ ldr(r0, iaddress(n));
 816 }
 817 
 818 void TemplateTable::aload_0() {
 819   aload_0_internal();
 820 }
 821 
 822 void TemplateTable::nofast_aload_0() {
 823   aload_0_internal(may_not_rewrite);
 824 }
 825 
 826 void TemplateTable::aload_0_internal(RewriteControl rc) {
 827   // According to bytecode histograms, the pairs:
 828   //
 829   // _aload_0, _fast_igetfield
 830   // _aload_0, _fast_agetfield
 831   // _aload_0, _fast_fgetfield
 832   //
 833   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 834   // _aload_0 bytecode checks if the next bytecode is either
 835   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 836   // rewrites the current bytecode into a pair bytecode; otherwise it
 837   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 838   // the pair check anymore.
 839   //
 840   // Note: If the next bytecode is _getfield, the rewrite must be
 841   //       delayed, otherwise we may miss an opportunity for a pair.
 842   //
 843   // Also rewrite frequent pairs
 844   //   aload_0, aload_1
 845   //   aload_0, iload_1
 846   // These bytecodes with a small amount of code are most profitable
 847   // to rewrite
 848   if (RewriteFrequentPairs && rc == may_rewrite) {
 849     Label rewrite, done;
 850     const Register bc = r4;
 851 
 852     // get next bytecode
 853     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 854 
 855     // if _getfield then wait with rewrite
 856     __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
 857     __ br(Assembler::EQ, done);
 858 
 859     // if _igetfield then rewrite to _fast_iaccess_0
 860     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 861     __ cmpw(r1, Bytecodes::_fast_igetfield);
 862     __ movw(bc, Bytecodes::_fast_iaccess_0);
 863     __ br(Assembler::EQ, rewrite);
 864 
 865     // if _agetfield then rewrite to _fast_aaccess_0
 866     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 867     __ cmpw(r1, Bytecodes::_fast_agetfield);
 868     __ movw(bc, Bytecodes::_fast_aaccess_0);
 869     __ br(Assembler::EQ, rewrite);
 870 
 871     // if _fgetfield then rewrite to _fast_faccess_0
 872     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 873     __ cmpw(r1, Bytecodes::_fast_fgetfield);
 874     __ movw(bc, Bytecodes::_fast_faccess_0);
 875     __ br(Assembler::EQ, rewrite);
 876 
 877     // else rewrite to _fast_aload0
 878     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 879     __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
 880 
 881     // rewrite
 882     // bc: new bytecode
 883     __ bind(rewrite);
 884     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 885 
 886     __ bind(done);
 887   }
 888 
 889   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 890   aload(0);
 891 }
 892 
 893 void TemplateTable::istore()
 894 {
 895   transition(itos, vtos);
 896   locals_index(r1);
 897   // FIXME: We're being very pernickerty here storing a jint in a
 898   // local with strw, which costs an extra instruction over what we'd
 899   // be able to do with a simple str.  We should just store the whole
 900   // word.
 901   __ lea(rscratch1, iaddress(r1));
 902   __ strw(r0, Address(rscratch1));
 903 }
 904 
 905 void TemplateTable::lstore()
 906 {
 907   transition(ltos, vtos);
 908   locals_index(r1);
 909   __ str(r0, laddress(r1, rscratch1, _masm));
 910 }
 911 
 912 void TemplateTable::fstore() {
 913   transition(ftos, vtos);
 914   locals_index(r1);
 915   __ lea(rscratch1, iaddress(r1));
 916   __ strs(v0, Address(rscratch1));
 917 }
 918 
 919 void TemplateTable::dstore() {
 920   transition(dtos, vtos);
 921   locals_index(r1);
 922   __ strd(v0, daddress(r1, rscratch1, _masm));
 923 }
 924 
 925 void TemplateTable::astore()
 926 {
 927   transition(vtos, vtos);
 928   __ pop_ptr(r0);
 929   locals_index(r1);
 930   __ str(r0, aaddress(r1));
 931 }
 932 
 933 void TemplateTable::wide_istore() {
 934   transition(vtos, vtos);
 935   __ pop_i();
 936   locals_index_wide(r1);
 937   __ lea(rscratch1, iaddress(r1));
 938   __ strw(r0, Address(rscratch1));
 939 }
 940 
 941 void TemplateTable::wide_lstore() {
 942   transition(vtos, vtos);
 943   __ pop_l();
 944   locals_index_wide(r1);
 945   __ str(r0, laddress(r1, rscratch1, _masm));
 946 }
 947 
 948 void TemplateTable::wide_fstore() {
 949   transition(vtos, vtos);
 950   __ pop_f();
 951   locals_index_wide(r1);
 952   __ lea(rscratch1, faddress(r1));
 953   __ strs(v0, rscratch1);
 954 }
 955 
 956 void TemplateTable::wide_dstore() {
 957   transition(vtos, vtos);
 958   __ pop_d();
 959   locals_index_wide(r1);
 960   __ strd(v0, daddress(r1, rscratch1, _masm));
 961 }
 962 
 963 void TemplateTable::wide_astore() {
 964   transition(vtos, vtos);
 965   __ pop_ptr(r0);
 966   locals_index_wide(r1);
 967   __ str(r0, aaddress(r1));
 968 }
 969 
 970 void TemplateTable::iastore() {
 971   transition(itos, vtos);
 972   __ pop_i(r1);
 973   __ pop_ptr(r3);
 974   // r0: value
 975   // r1: index
 976   // r3: array
 977   index_check(r3, r1); // prefer index in r1
 978   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
 979   __ strw(r0, Address(rscratch1,
 980                       arrayOopDesc::base_offset_in_bytes(T_INT)));
 981 }
 982 
 983 void TemplateTable::lastore() {
 984   transition(ltos, vtos);
 985   __ pop_i(r1);
 986   __ pop_ptr(r3);
 987   // r0: value
 988   // r1: index
 989   // r3: array
 990   index_check(r3, r1); // prefer index in r1
 991   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
 992   __ str(r0, Address(rscratch1,
 993                       arrayOopDesc::base_offset_in_bytes(T_LONG)));
 994 }
 995 
 996 void TemplateTable::fastore() {
 997   transition(ftos, vtos);
 998   __ pop_i(r1);
 999   __ pop_ptr(r3);
1000   // v0: value
1001   // r1:  index
1002   // r3:  array
1003   index_check(r3, r1); // prefer index in r1
1004   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1005   __ strs(v0, Address(rscratch1,
1006                       arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
1007 }
1008 
1009 void TemplateTable::dastore() {
1010   transition(dtos, vtos);
1011   __ pop_i(r1);
1012   __ pop_ptr(r3);
1013   // v0: value
1014   // r1:  index
1015   // r3:  array
1016   index_check(r3, r1); // prefer index in r1
1017   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1018   __ strd(v0, Address(rscratch1,
1019                       arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1020 }
1021 
1022 void TemplateTable::aastore() {
1023   Label is_null, ok_is_subtype, done;
1024   transition(vtos, vtos);
1025   // stack: ..., array, index, value
1026   __ ldr(r0, at_tos());    // value
1027   __ ldr(r2, at_tos_p1()); // index
1028   __ ldr(r3, at_tos_p2()); // array
1029 
1030   Address element_address(r4, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1031 
1032   index_check(r3, r2);     // kills r1
1033   __ lea(r4, Address(r3, r2, Address::uxtw(UseCompressedOops? 2 : 3)));
1034 
1035   // do array store check - check for NULL value first
1036   __ cbz(r0, is_null);
1037 
1038   // Move subklass into r1
1039   __ load_klass(r1, r0);
1040   // Move superklass into r0
1041   __ load_klass(r0, r3);
1042   __ ldr(r0, Address(r0,
1043                      ObjArrayKlass::element_klass_offset()));
1044   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1045 
1046   // Generate subtype check.  Blows r2, r5
1047   // Superklass in r0.  Subklass in r1.
1048   __ gen_subtype_check(r1, ok_is_subtype);
1049 
1050   // Come here on failure
1051   // object is at TOS
1052   __ b(Interpreter::_throw_ArrayStoreException_entry);
1053 
1054   // Come here on success
1055   __ bind(ok_is_subtype);
1056 
1057   // Get the value we will store
1058   __ ldr(r0, at_tos());
1059   // Now store using the appropriate barrier
1060   do_oop_store(_masm, element_address, r0, _bs->kind(), true);
1061   __ b(done);
1062 
1063   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1064   __ bind(is_null);
1065   __ profile_null_seen(r2);
1066 
1067   // Store a NULL
1068   do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
1069 
1070   // Pop stack arguments
1071   __ bind(done);
1072   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1073 }
1074 
1075 void TemplateTable::bastore()
1076 {
1077   transition(itos, vtos);
1078   __ pop_i(r1);
1079   __ pop_ptr(r3);
1080   // r0: value
1081   // r1: index
1082   // r3: array
1083   index_check(r3, r1); // prefer index in r1
1084 
1085   // Need to check whether array is boolean or byte
1086   // since both types share the bastore bytecode.
1087   __ load_klass(r2, r3);
1088   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1089   int diffbit_index = exact_log2(Klass::layout_helper_boolean_diffbit());
1090   Label L_skip;
1091   __ tbz(r2, diffbit_index, L_skip);
1092   __ andw(r0, r0, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1093   __ bind(L_skip);
1094 
1095   __ lea(rscratch1, Address(r3, r1, Address::uxtw(0)));
1096   __ strb(r0, Address(rscratch1,
1097                       arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1098 }
1099 
1100 void TemplateTable::castore()
1101 {
1102   transition(itos, vtos);
1103   __ pop_i(r1);
1104   __ pop_ptr(r3);
1105   // r0: value
1106   // r1: index
1107   // r3: array
1108   index_check(r3, r1); // prefer index in r1
1109   __ lea(rscratch1, Address(r3, r1, Address::uxtw(1)));
1110   __ strh(r0, Address(rscratch1,
1111                       arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1112 }
1113 
1114 void TemplateTable::sastore()
1115 {
1116   castore();
1117 }
1118 
1119 void TemplateTable::istore(int n)
1120 {
1121   transition(itos, vtos);
1122   __ str(r0, iaddress(n));
1123 }
1124 
1125 void TemplateTable::lstore(int n)
1126 {
1127   transition(ltos, vtos);
1128   __ str(r0, laddress(n));
1129 }
1130 
1131 void TemplateTable::fstore(int n)
1132 {
1133   transition(ftos, vtos);
1134   __ strs(v0, faddress(n));
1135 }
1136 
1137 void TemplateTable::dstore(int n)
1138 {
1139   transition(dtos, vtos);
1140   __ strd(v0, daddress(n));
1141 }
1142 
1143 void TemplateTable::astore(int n)
1144 {
1145   transition(vtos, vtos);
1146   __ pop_ptr(r0);
1147   __ str(r0, iaddress(n));
1148 }
1149 
1150 void TemplateTable::pop()
1151 {
1152   transition(vtos, vtos);
1153   __ add(esp, esp, Interpreter::stackElementSize);
1154 }
1155 
1156 void TemplateTable::pop2()
1157 {
1158   transition(vtos, vtos);
1159   __ add(esp, esp, 2 * Interpreter::stackElementSize);
1160 }
1161 
1162 void TemplateTable::dup()
1163 {
1164   transition(vtos, vtos);
1165   __ ldr(r0, Address(esp, 0));
1166   __ push(r0);
1167   // stack: ..., a, a
1168 }
1169 
1170 void TemplateTable::dup_x1()
1171 {
1172   transition(vtos, vtos);
1173   // stack: ..., a, b
1174   __ ldr(r0, at_tos());  // load b
1175   __ ldr(r2, at_tos_p1());  // load a
1176   __ str(r0, at_tos_p1());  // store b
1177   __ str(r2, at_tos());  // store a
1178   __ push(r0);                  // push b
1179   // stack: ..., b, a, b
1180 }
1181 
1182 void TemplateTable::dup_x2()
1183 {
1184   transition(vtos, vtos);
1185   // stack: ..., a, b, c
1186   __ ldr(r0, at_tos());  // load c
1187   __ ldr(r2, at_tos_p2());  // load a
1188   __ str(r0, at_tos_p2());  // store c in a
1189   __ push(r0);      // push c
1190   // stack: ..., c, b, c, c
1191   __ ldr(r0, at_tos_p2());  // load b
1192   __ str(r2, at_tos_p2());  // store a in b
1193   // stack: ..., c, a, c, c
1194   __ str(r0, at_tos_p1());  // store b in c
1195   // stack: ..., c, a, b, c
1196 }
1197 
1198 void TemplateTable::dup2()
1199 {
1200   transition(vtos, vtos);
1201   // stack: ..., a, b
1202   __ ldr(r0, at_tos_p1());  // load a
1203   __ push(r0);                  // push a
1204   __ ldr(r0, at_tos_p1());  // load b
1205   __ push(r0);                  // push b
1206   // stack: ..., a, b, a, b
1207 }
1208 
1209 void TemplateTable::dup2_x1()
1210 {
1211   transition(vtos, vtos);
1212   // stack: ..., a, b, c
1213   __ ldr(r2, at_tos());  // load c
1214   __ ldr(r0, at_tos_p1());  // load b
1215   __ push(r0);                  // push b
1216   __ push(r2);                  // push c
1217   // stack: ..., a, b, c, b, c
1218   __ str(r2, at_tos_p3());  // store c in b
1219   // stack: ..., a, c, c, b, c
1220   __ ldr(r2, at_tos_p4());  // load a
1221   __ str(r2, at_tos_p2());  // store a in 2nd c
1222   // stack: ..., a, c, a, b, c
1223   __ str(r0, at_tos_p4());  // store b in a
1224   // stack: ..., b, c, a, b, c
1225 }
1226 
1227 void TemplateTable::dup2_x2()
1228 {
1229   transition(vtos, vtos);
1230   // stack: ..., a, b, c, d
1231   __ ldr(r2, at_tos());  // load d
1232   __ ldr(r0, at_tos_p1());  // load c
1233   __ push(r0)            ;      // push c
1234   __ push(r2);                  // push d
1235   // stack: ..., a, b, c, d, c, d
1236   __ ldr(r0, at_tos_p4());  // load b
1237   __ str(r0, at_tos_p2());  // store b in d
1238   __ str(r2, at_tos_p4());  // store d in b
1239   // stack: ..., a, d, c, b, c, d
1240   __ ldr(r2, at_tos_p5());  // load a
1241   __ ldr(r0, at_tos_p3());  // load c
1242   __ str(r2, at_tos_p3());  // store a in c
1243   __ str(r0, at_tos_p5());  // store c in a
1244   // stack: ..., c, d, a, b, c, d
1245 }
1246 
1247 void TemplateTable::swap()
1248 {
1249   transition(vtos, vtos);
1250   // stack: ..., a, b
1251   __ ldr(r2, at_tos_p1());  // load a
1252   __ ldr(r0, at_tos());  // load b
1253   __ str(r2, at_tos());  // store a in b
1254   __ str(r0, at_tos_p1());  // store b in a
1255   // stack: ..., b, a
1256 }
1257 
1258 void TemplateTable::iop2(Operation op)
1259 {
1260   transition(itos, itos);
1261   // r0 <== r1 op r0
1262   __ pop_i(r1);
1263   switch (op) {
1264   case add  : __ addw(r0, r1, r0); break;
1265   case sub  : __ subw(r0, r1, r0); break;
1266   case mul  : __ mulw(r0, r1, r0); break;
1267   case _and : __ andw(r0, r1, r0); break;
1268   case _or  : __ orrw(r0, r1, r0); break;
1269   case _xor : __ eorw(r0, r1, r0); break;
1270   case shl  : __ lslvw(r0, r1, r0); break;
1271   case shr  : __ asrvw(r0, r1, r0); break;
1272   case ushr : __ lsrvw(r0, r1, r0);break;
1273   default   : ShouldNotReachHere();
1274   }
1275 }
1276 
1277 void TemplateTable::lop2(Operation op)
1278 {
1279   transition(ltos, ltos);
1280   // r0 <== r1 op r0
1281   __ pop_l(r1);
1282   switch (op) {
1283   case add  : __ add(r0, r1, r0); break;
1284   case sub  : __ sub(r0, r1, r0); break;
1285   case mul  : __ mul(r0, r1, r0); break;
1286   case _and : __ andr(r0, r1, r0); break;
1287   case _or  : __ orr(r0, r1, r0); break;
1288   case _xor : __ eor(r0, r1, r0); break;
1289   default   : ShouldNotReachHere();
1290   }
1291 }
1292 
1293 void TemplateTable::idiv()
1294 {
1295   transition(itos, itos);
1296   // explicitly check for div0
1297   Label no_div0;
1298   __ cbnzw(r0, no_div0);
1299   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1300   __ br(rscratch1);
1301   __ bind(no_div0);
1302   __ pop_i(r1);
1303   // r0 <== r1 idiv r0
1304   __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1305 }
1306 
1307 void TemplateTable::irem()
1308 {
1309   transition(itos, itos);
1310   // explicitly check for div0
1311   Label no_div0;
1312   __ cbnzw(r0, no_div0);
1313   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1314   __ br(rscratch1);
1315   __ bind(no_div0);
1316   __ pop_i(r1);
1317   // r0 <== r1 irem r0
1318   __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1319 }
1320 
1321 void TemplateTable::lmul()
1322 {
1323   transition(ltos, ltos);
1324   __ pop_l(r1);
1325   __ mul(r0, r0, r1);
1326 }
1327 
1328 void TemplateTable::ldiv()
1329 {
1330   transition(ltos, ltos);
1331   // explicitly check for div0
1332   Label no_div0;
1333   __ cbnz(r0, no_div0);
1334   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1335   __ br(rscratch1);
1336   __ bind(no_div0);
1337   __ pop_l(r1);
1338   // r0 <== r1 ldiv r0
1339   __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1340 }
1341 
1342 void TemplateTable::lrem()
1343 {
1344   transition(ltos, ltos);
1345   // explicitly check for div0
1346   Label no_div0;
1347   __ cbnz(r0, no_div0);
1348   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1349   __ br(rscratch1);
1350   __ bind(no_div0);
1351   __ pop_l(r1);
1352   // r0 <== r1 lrem r0
1353   __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1354 }
1355 
1356 void TemplateTable::lshl()
1357 {
1358   transition(itos, ltos);
1359   // shift count is in r0
1360   __ pop_l(r1);
1361   __ lslv(r0, r1, r0);
1362 }
1363 
1364 void TemplateTable::lshr()
1365 {
1366   transition(itos, ltos);
1367   // shift count is in r0
1368   __ pop_l(r1);
1369   __ asrv(r0, r1, r0);
1370 }
1371 
1372 void TemplateTable::lushr()
1373 {
1374   transition(itos, ltos);
1375   // shift count is in r0
1376   __ pop_l(r1);
1377   __ lsrv(r0, r1, r0);
1378 }
1379 
1380 void TemplateTable::fop2(Operation op)
1381 {
1382   transition(ftos, ftos);
1383   switch (op) {
1384   case add:
1385     // n.b. use ldrd because this is a 64 bit slot
1386     __ pop_f(v1);
1387     __ fadds(v0, v1, v0);
1388     break;
1389   case sub:
1390     __ pop_f(v1);
1391     __ fsubs(v0, v1, v0);
1392     break;
1393   case mul:
1394     __ pop_f(v1);
1395     __ fmuls(v0, v1, v0);
1396     break;
1397   case div:
1398     __ pop_f(v1);
1399     __ fdivs(v0, v1, v0);
1400     break;
1401   case rem:
1402     __ fmovs(v1, v0);
1403     __ pop_f(v0);
1404     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::frem),
1405                          0, 2, MacroAssembler::ret_type_float);
1406     break;
1407   default:
1408     ShouldNotReachHere();
1409     break;
1410   }
1411 }
1412 
1413 void TemplateTable::dop2(Operation op)
1414 {
1415   transition(dtos, dtos);
1416   switch (op) {
1417   case add:
1418     // n.b. use ldrd because this is a 64 bit slot
1419     __ pop_d(v1);
1420     __ faddd(v0, v1, v0);
1421     break;
1422   case sub:
1423     __ pop_d(v1);
1424     __ fsubd(v0, v1, v0);
1425     break;
1426   case mul:
1427     __ pop_d(v1);
1428     __ fmuld(v0, v1, v0);
1429     break;
1430   case div:
1431     __ pop_d(v1);
1432     __ fdivd(v0, v1, v0);
1433     break;
1434   case rem:
1435     __ fmovd(v1, v0);
1436     __ pop_d(v0);
1437     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::drem),
1438                          0, 2, MacroAssembler::ret_type_double);
1439     break;
1440   default:
1441     ShouldNotReachHere();
1442     break;
1443   }
1444 }
1445 
1446 void TemplateTable::ineg()
1447 {
1448   transition(itos, itos);
1449   __ negw(r0, r0);
1450 
1451 }
1452 
1453 void TemplateTable::lneg()
1454 {
1455   transition(ltos, ltos);
1456   __ neg(r0, r0);
1457 }
1458 
1459 void TemplateTable::fneg()
1460 {
1461   transition(ftos, ftos);
1462   __ fnegs(v0, v0);
1463 }
1464 
1465 void TemplateTable::dneg()
1466 {
1467   transition(dtos, dtos);
1468   __ fnegd(v0, v0);
1469 }
1470 
1471 void TemplateTable::iinc()
1472 {
1473   transition(vtos, vtos);
1474   __ load_signed_byte(r1, at_bcp(2)); // get constant
1475   locals_index(r2);
1476   __ ldr(r0, iaddress(r2));
1477   __ addw(r0, r0, r1);
1478   __ str(r0, iaddress(r2));
1479 }
1480 
1481 void TemplateTable::wide_iinc()
1482 {
1483   transition(vtos, vtos);
1484   // __ mov(r1, zr);
1485   __ ldrw(r1, at_bcp(2)); // get constant and index
1486   __ rev16(r1, r1);
1487   __ ubfx(r2, r1, 0, 16);
1488   __ neg(r2, r2);
1489   __ sbfx(r1, r1, 16, 16);
1490   __ ldr(r0, iaddress(r2));
1491   __ addw(r0, r0, r1);
1492   __ str(r0, iaddress(r2));
1493 }
1494 
1495 void TemplateTable::convert()
1496 {
1497   // Checking
1498 #ifdef ASSERT
1499   {
1500     TosState tos_in  = ilgl;
1501     TosState tos_out = ilgl;
1502     switch (bytecode()) {
1503     case Bytecodes::_i2l: // fall through
1504     case Bytecodes::_i2f: // fall through
1505     case Bytecodes::_i2d: // fall through
1506     case Bytecodes::_i2b: // fall through
1507     case Bytecodes::_i2c: // fall through
1508     case Bytecodes::_i2s: tos_in = itos; break;
1509     case Bytecodes::_l2i: // fall through
1510     case Bytecodes::_l2f: // fall through
1511     case Bytecodes::_l2d: tos_in = ltos; break;
1512     case Bytecodes::_f2i: // fall through
1513     case Bytecodes::_f2l: // fall through
1514     case Bytecodes::_f2d: tos_in = ftos; break;
1515     case Bytecodes::_d2i: // fall through
1516     case Bytecodes::_d2l: // fall through
1517     case Bytecodes::_d2f: tos_in = dtos; break;
1518     default             : ShouldNotReachHere();
1519     }
1520     switch (bytecode()) {
1521     case Bytecodes::_l2i: // fall through
1522     case Bytecodes::_f2i: // fall through
1523     case Bytecodes::_d2i: // fall through
1524     case Bytecodes::_i2b: // fall through
1525     case Bytecodes::_i2c: // fall through
1526     case Bytecodes::_i2s: tos_out = itos; break;
1527     case Bytecodes::_i2l: // fall through
1528     case Bytecodes::_f2l: // fall through
1529     case Bytecodes::_d2l: tos_out = ltos; break;
1530     case Bytecodes::_i2f: // fall through
1531     case Bytecodes::_l2f: // fall through
1532     case Bytecodes::_d2f: tos_out = ftos; break;
1533     case Bytecodes::_i2d: // fall through
1534     case Bytecodes::_l2d: // fall through
1535     case Bytecodes::_f2d: tos_out = dtos; break;
1536     default             : ShouldNotReachHere();
1537     }
1538     transition(tos_in, tos_out);
1539   }
1540 #endif // ASSERT
1541   // static const int64_t is_nan = 0x8000000000000000L;
1542 
1543   // Conversion
1544   switch (bytecode()) {
1545   case Bytecodes::_i2l:
1546     __ sxtw(r0, r0);
1547     break;
1548   case Bytecodes::_i2f:
1549     __ scvtfws(v0, r0);
1550     break;
1551   case Bytecodes::_i2d:
1552     __ scvtfwd(v0, r0);
1553     break;
1554   case Bytecodes::_i2b:
1555     __ sxtbw(r0, r0);
1556     break;
1557   case Bytecodes::_i2c:
1558     __ uxthw(r0, r0);
1559     break;
1560   case Bytecodes::_i2s:
1561     __ sxthw(r0, r0);
1562     break;
1563   case Bytecodes::_l2i:
1564     __ uxtw(r0, r0);
1565     break;
1566   case Bytecodes::_l2f:
1567     __ scvtfs(v0, r0);
1568     break;
1569   case Bytecodes::_l2d:
1570     __ scvtfd(v0, r0);
1571     break;
1572   case Bytecodes::_f2i:
1573   {
1574     Label L_Okay;
1575     __ clear_fpsr();
1576     __ fcvtzsw(r0, v0);
1577     __ get_fpsr(r1);
1578     __ cbzw(r1, L_Okay);
1579     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2i),
1580                          0, 1, MacroAssembler::ret_type_integral);
1581     __ bind(L_Okay);
1582   }
1583     break;
1584   case Bytecodes::_f2l:
1585   {
1586     Label L_Okay;
1587     __ clear_fpsr();
1588     __ fcvtzs(r0, v0);
1589     __ get_fpsr(r1);
1590     __ cbzw(r1, L_Okay);
1591     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2l),
1592                          0, 1, MacroAssembler::ret_type_integral);
1593     __ bind(L_Okay);
1594   }
1595     break;
1596   case Bytecodes::_f2d:
1597     __ fcvts(v0, v0);
1598     break;
1599   case Bytecodes::_d2i:
1600   {
1601     Label L_Okay;
1602     __ clear_fpsr();
1603     __ fcvtzdw(r0, v0);
1604     __ get_fpsr(r1);
1605     __ cbzw(r1, L_Okay);
1606     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2i),
1607                          0, 1, MacroAssembler::ret_type_integral);
1608     __ bind(L_Okay);
1609   }
1610     break;
1611   case Bytecodes::_d2l:
1612   {
1613     Label L_Okay;
1614     __ clear_fpsr();
1615     __ fcvtzd(r0, v0);
1616     __ get_fpsr(r1);
1617     __ cbzw(r1, L_Okay);
1618     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2l),
1619                          0, 1, MacroAssembler::ret_type_integral);
1620     __ bind(L_Okay);
1621   }
1622     break;
1623   case Bytecodes::_d2f:
1624     __ fcvtd(v0, v0);
1625     break;
1626   default:
1627     ShouldNotReachHere();
1628   }
1629 }
1630 
1631 void TemplateTable::lcmp()
1632 {
1633   transition(ltos, itos);
1634   Label done;
1635   __ pop_l(r1);
1636   __ cmp(r1, r0);
1637   __ mov(r0, (u_int64_t)-1L);
1638   __ br(Assembler::LT, done);
1639   // __ mov(r0, 1UL);
1640   // __ csel(r0, r0, zr, Assembler::NE);
1641   // and here is a faster way
1642   __ csinc(r0, zr, zr, Assembler::EQ);
1643   __ bind(done);
1644 }
1645 
1646 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1647 {
1648   Label done;
1649   if (is_float) {
1650     // XXX get rid of pop here, use ... reg, mem32
1651     __ pop_f(v1);
1652     __ fcmps(v1, v0);
1653   } else {
1654     // XXX get rid of pop here, use ... reg, mem64
1655     __ pop_d(v1);
1656     __ fcmpd(v1, v0);
1657   }
1658   if (unordered_result < 0) {
1659     // we want -1 for unordered or less than, 0 for equal and 1 for
1660     // greater than.
1661     __ mov(r0, (u_int64_t)-1L);
1662     // for FP LT tests less than or unordered
1663     __ br(Assembler::LT, done);
1664     // install 0 for EQ otherwise 1
1665     __ csinc(r0, zr, zr, Assembler::EQ);
1666   } else {
1667     // we want -1 for less than, 0 for equal and 1 for unordered or
1668     // greater than.
1669     __ mov(r0, 1L);
1670     // for FP HI tests greater than or unordered
1671     __ br(Assembler::HI, done);
1672     // install 0 for EQ otherwise ~0
1673     __ csinv(r0, zr, zr, Assembler::EQ);
1674 
1675   }
1676   __ bind(done);
1677 }
1678 
1679 void TemplateTable::branch(bool is_jsr, bool is_wide)
1680 {
1681   // We might be moving to a safepoint.  The thread which calls
1682   // Interpreter::notice_safepoints() will effectively flush its cache
1683   // when it makes a system call, but we need to do something to
1684   // ensure that we see the changed dispatch table.
1685   __ membar(MacroAssembler::LoadLoad);
1686 
1687   __ profile_taken_branch(r0, r1);
1688   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1689                              InvocationCounter::counter_offset();
1690   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1691                               InvocationCounter::counter_offset();
1692 
1693   // load branch displacement
1694   if (!is_wide) {
1695     __ ldrh(r2, at_bcp(1));
1696     __ rev16(r2, r2);
1697     // sign extend the 16 bit value in r2
1698     __ sbfm(r2, r2, 0, 15);
1699   } else {
1700     __ ldrw(r2, at_bcp(1));
1701     __ revw(r2, r2);
1702     // sign extend the 32 bit value in r2
1703     __ sbfm(r2, r2, 0, 31);
1704   }
1705 
1706   // Handle all the JSR stuff here, then exit.
1707   // It's much shorter and cleaner than intermingling with the non-JSR
1708   // normal-branch stuff occurring below.
1709 
1710   if (is_jsr) {
1711     // Pre-load the next target bytecode into rscratch1
1712     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1713     // compute return address as bci
1714     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1715     __ add(rscratch2, rscratch2,
1716            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1717     __ sub(r1, rbcp, rscratch2);
1718     __ push_i(r1);
1719     // Adjust the bcp by the 16-bit displacement in r2
1720     __ add(rbcp, rbcp, r2);
1721     __ dispatch_only(vtos);
1722     return;
1723   }
1724 
1725   // Normal (non-jsr) branch handling
1726 
1727   // Adjust the bcp by the displacement in r2
1728   __ add(rbcp, rbcp, r2);
1729 
1730   assert(UseLoopCounter || !UseOnStackReplacement,
1731          "on-stack-replacement requires loop counters");
1732   Label backedge_counter_overflow;
1733   Label profile_method;
1734   Label dispatch;
1735   if (UseLoopCounter) {
1736     // increment backedge counter for backward branches
1737     // r0: MDO
1738     // w1: MDO bumped taken-count
1739     // r2: target offset
1740     __ cmp(r2, zr);
1741     __ br(Assembler::GT, dispatch); // count only if backward branch
1742 
1743     // ECN: FIXME: This code smells
1744     // check if MethodCounters exists
1745     Label has_counters;
1746     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1747     __ cbnz(rscratch1, has_counters);
1748     __ push(r0);
1749     __ push(r1);
1750     __ push(r2);
1751     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1752             InterpreterRuntime::build_method_counters), rmethod);
1753     __ pop(r2);
1754     __ pop(r1);
1755     __ pop(r0);
1756     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1757     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1758     __ bind(has_counters);
1759 
1760     if (TieredCompilation) {
1761       Label no_mdo;
1762       int increment = InvocationCounter::count_increment;
1763       if (ProfileInterpreter) {
1764         // Are we profiling?
1765         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1766         __ cbz(r1, no_mdo);
1767         // Increment the MDO backedge counter
1768         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1769                                            in_bytes(InvocationCounter::counter_offset()));
1770         const Address mask(r1, in_bytes(MethodData::backedge_mask_offset()));
1771         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1772                                    r0, rscratch1, false, Assembler::EQ, &backedge_counter_overflow);
1773         __ b(dispatch);
1774       }
1775       __ bind(no_mdo);
1776       // Increment backedge counter in MethodCounters*
1777       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1778       const Address mask(rscratch1, in_bytes(MethodCounters::backedge_mask_offset()));
1779       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1780                                  r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
1781     } else { // not TieredCompilation
1782       // increment counter
1783       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1784       __ ldrw(r0, Address(rscratch2, be_offset));        // load backedge counter
1785       __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1786       __ strw(rscratch1, Address(rscratch2, be_offset));        // store counter
1787 
1788       __ ldrw(r0, Address(rscratch2, inv_offset));    // load invocation counter
1789       __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1790       __ addw(r0, r0, rscratch1);        // add both counters
1791 
1792       if (ProfileInterpreter) {
1793         // Test to see if we should create a method data oop
1794         __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_profile_limit_offset())));
1795         __ cmpw(r0, rscratch1);
1796         __ br(Assembler::LT, dispatch);
1797 
1798         // if no method data exists, go to profile method
1799         __ test_method_data_pointer(r0, profile_method);
1800 
1801         if (UseOnStackReplacement) {
1802           // check for overflow against w1 which is the MDO taken count
1803           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1804           __ cmpw(r1, rscratch1);
1805           __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1806 
1807           // When ProfileInterpreter is on, the backedge_count comes
1808           // from the MethodData*, which value does not get reset on
1809           // the call to frequency_counter_overflow().  To avoid
1810           // excessive calls to the overflow routine while the method is
1811           // being compiled, add a second test to make sure the overflow
1812           // function is called only once every overflow_frequency.
1813           const int overflow_frequency = 1024;
1814           __ andsw(r1, r1, overflow_frequency - 1);
1815           __ br(Assembler::EQ, backedge_counter_overflow);
1816 
1817         }
1818       } else {
1819         if (UseOnStackReplacement) {
1820           // check for overflow against w0, which is the sum of the
1821           // counters
1822           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1823           __ cmpw(r0, rscratch1);
1824           __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1825         }
1826       }
1827     }
1828   }
1829   __ bind(dispatch);
1830 
1831   // Pre-load the next target bytecode into rscratch1
1832   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1833 
1834   // continue with the bytecode @ target
1835   // rscratch1: target bytecode
1836   // rbcp: target bcp
1837   __ dispatch_only(vtos);
1838 
1839   if (UseLoopCounter) {
1840     if (ProfileInterpreter) {
1841       // Out-of-line code to allocate method data oop.
1842       __ bind(profile_method);
1843       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1844       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1845       __ set_method_data_pointer_for_bcp();
1846       __ b(dispatch);
1847     }
1848 
1849     if (TieredCompilation || UseOnStackReplacement) {
1850       // invocation counter overflow
1851       __ bind(backedge_counter_overflow);
1852       __ neg(r2, r2);
1853       __ add(r2, r2, rbcp);     // branch bcp
1854       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1855       __ call_VM(noreg,
1856                  CAST_FROM_FN_PTR(address,
1857                                   InterpreterRuntime::frequency_counter_overflow),
1858                  r2);
1859       if (!UseOnStackReplacement)
1860         __ b(dispatch);
1861     }
1862 
1863     if (UseOnStackReplacement) {
1864       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1865 
1866       // r0: osr nmethod (osr ok) or NULL (osr not possible)
1867       // w1: target bytecode
1868       // r2: scratch
1869       __ cbz(r0, dispatch);     // test result -- no osr if null
1870       // nmethod may have been invalidated (VM may block upon call_VM return)
1871       __ ldrb(r2, Address(r0, nmethod::state_offset()));
1872       if (nmethod::in_use != 0)
1873         __ sub(r2, r2, nmethod::in_use);
1874       __ cbnz(r2, dispatch);
1875 
1876       // We have the address of an on stack replacement routine in r0
1877       // We need to prepare to execute the OSR method. First we must
1878       // migrate the locals and monitors off of the stack.
1879 
1880       __ mov(r19, r0);                             // save the nmethod
1881 
1882       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1883 
1884       // r0 is OSR buffer, move it to expected parameter location
1885       __ mov(j_rarg0, r0);
1886 
1887       // remove activation
1888       // get sender esp
1889       __ ldr(esp,
1890           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
1891       // remove frame anchor
1892       __ leave();
1893       // Ensure compiled code always sees stack at proper alignment
1894       __ andr(sp, esp, -16);
1895 
1896       // and begin the OSR nmethod
1897       __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
1898       __ br(rscratch1);
1899     }
1900   }
1901 }
1902 
1903 
1904 void TemplateTable::if_0cmp(Condition cc)
1905 {
1906   transition(itos, vtos);
1907   // assume branch is more often taken than not (loops use backward branches)
1908   Label not_taken;
1909   if (cc == equal)
1910     __ cbnzw(r0, not_taken);
1911   else if (cc == not_equal)
1912     __ cbzw(r0, not_taken);
1913   else {
1914     __ andsw(zr, r0, r0);
1915     __ br(j_not(cc), not_taken);
1916   }
1917 
1918   branch(false, false);
1919   __ bind(not_taken);
1920   __ profile_not_taken_branch(r0);
1921 }
1922 
1923 void TemplateTable::if_icmp(Condition cc)
1924 {
1925   transition(itos, vtos);
1926   // assume branch is more often taken than not (loops use backward branches)
1927   Label not_taken;
1928   __ pop_i(r1);
1929   __ cmpw(r1, r0, Assembler::LSL);
1930   __ br(j_not(cc), not_taken);
1931   branch(false, false);
1932   __ bind(not_taken);
1933   __ profile_not_taken_branch(r0);
1934 }
1935 
1936 void TemplateTable::if_nullcmp(Condition cc)
1937 {
1938   transition(atos, vtos);
1939   // assume branch is more often taken than not (loops use backward branches)
1940   Label not_taken;
1941   if (cc == equal)
1942     __ cbnz(r0, not_taken);
1943   else
1944     __ cbz(r0, not_taken);
1945   branch(false, false);
1946   __ bind(not_taken);
1947   __ profile_not_taken_branch(r0);
1948 }
1949 
1950 void TemplateTable::if_acmp(Condition cc)
1951 {
1952   transition(atos, vtos);
1953   // assume branch is more often taken than not (loops use backward branches)
1954   Label not_taken;
1955   __ pop_ptr(r1);
1956   __ cmp(r1, r0);
1957   __ br(j_not(cc), not_taken);
1958   branch(false, false);
1959   __ bind(not_taken);
1960   __ profile_not_taken_branch(r0);
1961 }
1962 
1963 void TemplateTable::ret() {
1964   transition(vtos, vtos);
1965   // We might be moving to a safepoint.  The thread which calls
1966   // Interpreter::notice_safepoints() will effectively flush its cache
1967   // when it makes a system call, but we need to do something to
1968   // ensure that we see the changed dispatch table.
1969   __ membar(MacroAssembler::LoadLoad);
1970 
1971   locals_index(r1);
1972   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1973   __ profile_ret(r1, r2);
1974   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1975   __ lea(rbcp, Address(rbcp, r1));
1976   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
1977   __ dispatch_next(vtos);
1978 }
1979 
1980 void TemplateTable::wide_ret() {
1981   transition(vtos, vtos);
1982   locals_index_wide(r1);
1983   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1984   __ profile_ret(r1, r2);
1985   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1986   __ lea(rbcp, Address(rbcp, r1));
1987   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
1988   __ dispatch_next(vtos);
1989 }
1990 
1991 
1992 void TemplateTable::tableswitch() {
1993   Label default_case, continue_execution;
1994   transition(itos, vtos);
1995   // align rbcp
1996   __ lea(r1, at_bcp(BytesPerInt));
1997   __ andr(r1, r1, -BytesPerInt);
1998   // load lo & hi
1999   __ ldrw(r2, Address(r1, BytesPerInt));
2000   __ ldrw(r3, Address(r1, 2 * BytesPerInt));
2001   __ rev32(r2, r2);
2002   __ rev32(r3, r3);
2003   // check against lo & hi
2004   __ cmpw(r0, r2);
2005   __ br(Assembler::LT, default_case);
2006   __ cmpw(r0, r3);
2007   __ br(Assembler::GT, default_case);
2008   // lookup dispatch offset
2009   __ subw(r0, r0, r2);
2010   __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2011   __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2012   __ profile_switch_case(r0, r1, r2);
2013   // continue execution
2014   __ bind(continue_execution);
2015   __ rev32(r3, r3);
2016   __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2017   __ add(rbcp, rbcp, r3, ext::sxtw);
2018   __ dispatch_only(vtos);
2019   // handle default
2020   __ bind(default_case);
2021   __ profile_switch_default(r0);
2022   __ ldrw(r3, Address(r1, 0));
2023   __ b(continue_execution);
2024 }
2025 
2026 void TemplateTable::lookupswitch() {
2027   transition(itos, itos);
2028   __ stop("lookupswitch bytecode should have been rewritten");
2029 }
2030 
2031 void TemplateTable::fast_linearswitch() {
2032   transition(itos, vtos);
2033   Label loop_entry, loop, found, continue_execution;
2034   // bswap r0 so we can avoid bswapping the table entries
2035   __ rev32(r0, r0);
2036   // align rbcp
2037   __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2038                                     // this instruction (change offsets
2039                                     // below)
2040   __ andr(r19, r19, -BytesPerInt);
2041   // set counter
2042   __ ldrw(r1, Address(r19, BytesPerInt));
2043   __ rev32(r1, r1);
2044   __ b(loop_entry);
2045   // table search
2046   __ bind(loop);
2047   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2048   __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2049   __ cmpw(r0, rscratch1);
2050   __ br(Assembler::EQ, found);
2051   __ bind(loop_entry);
2052   __ subs(r1, r1, 1);
2053   __ br(Assembler::PL, loop);
2054   // default case
2055   __ profile_switch_default(r0);
2056   __ ldrw(r3, Address(r19, 0));
2057   __ b(continue_execution);
2058   // entry found -> get offset
2059   __ bind(found);
2060   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2061   __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2062   __ profile_switch_case(r1, r0, r19);
2063   // continue execution
2064   __ bind(continue_execution);
2065   __ rev32(r3, r3);
2066   __ add(rbcp, rbcp, r3, ext::sxtw);
2067   __ ldrb(rscratch1, Address(rbcp, 0));
2068   __ dispatch_only(vtos);
2069 }
2070 
2071 void TemplateTable::fast_binaryswitch() {
2072   transition(itos, vtos);
2073   // Implementation using the following core algorithm:
2074   //
2075   // int binary_search(int key, LookupswitchPair* array, int n) {
2076   //   // Binary search according to "Methodik des Programmierens" by
2077   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2078   //   int i = 0;
2079   //   int j = n;
2080   //   while (i+1 < j) {
2081   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2082   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2083   //     // where a stands for the array and assuming that the (inexisting)
2084   //     // element a[n] is infinitely big.
2085   //     int h = (i + j) >> 1;
2086   //     // i < h < j
2087   //     if (key < array[h].fast_match()) {
2088   //       j = h;
2089   //     } else {
2090   //       i = h;
2091   //     }
2092   //   }
2093   //   // R: a[i] <= key < a[i+1] or Q
2094   //   // (i.e., if key is within array, i is the correct index)
2095   //   return i;
2096   // }
2097 
2098   // Register allocation
2099   const Register key   = r0; // already set (tosca)
2100   const Register array = r1;
2101   const Register i     = r2;
2102   const Register j     = r3;
2103   const Register h     = rscratch1;
2104   const Register temp  = rscratch2;
2105 
2106   // Find array start
2107   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2108                                           // get rid of this
2109                                           // instruction (change
2110                                           // offsets below)
2111   __ andr(array, array, -BytesPerInt);
2112 
2113   // Initialize i & j
2114   __ mov(i, 0);                            // i = 0;
2115   __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2116 
2117   // Convert j into native byteordering
2118   __ rev32(j, j);
2119 
2120   // And start
2121   Label entry;
2122   __ b(entry);
2123 
2124   // binary search loop
2125   {
2126     Label loop;
2127     __ bind(loop);
2128     // int h = (i + j) >> 1;
2129     __ addw(h, i, j);                           // h = i + j;
2130     __ lsrw(h, h, 1);                                   // h = (i + j) >> 1;
2131     // if (key < array[h].fast_match()) {
2132     //   j = h;
2133     // } else {
2134     //   i = h;
2135     // }
2136     // Convert array[h].match to native byte-ordering before compare
2137     __ ldr(temp, Address(array, h, Address::lsl(3)));
2138     __ rev32(temp, temp);
2139     __ cmpw(key, temp);
2140     // j = h if (key <  array[h].fast_match())
2141     __ csel(j, h, j, Assembler::LT);
2142     // i = h if (key >= array[h].fast_match())
2143     __ csel(i, h, i, Assembler::GE);
2144     // while (i+1 < j)
2145     __ bind(entry);
2146     __ addw(h, i, 1);          // i+1
2147     __ cmpw(h, j);             // i+1 < j
2148     __ br(Assembler::LT, loop);
2149   }
2150 
2151   // end of binary search, result index is i (must check again!)
2152   Label default_case;
2153   // Convert array[i].match to native byte-ordering before compare
2154   __ ldr(temp, Address(array, i, Address::lsl(3)));
2155   __ rev32(temp, temp);
2156   __ cmpw(key, temp);
2157   __ br(Assembler::NE, default_case);
2158 
2159   // entry found -> j = offset
2160   __ add(j, array, i, ext::uxtx, 3);
2161   __ ldrw(j, Address(j, BytesPerInt));
2162   __ profile_switch_case(i, key, array);
2163   __ rev32(j, j);
2164   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2165   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2166   __ dispatch_only(vtos);
2167 
2168   // default case -> j = default offset
2169   __ bind(default_case);
2170   __ profile_switch_default(i);
2171   __ ldrw(j, Address(array, -2 * BytesPerInt));
2172   __ rev32(j, j);
2173   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2174   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2175   __ dispatch_only(vtos);
2176 }
2177 
2178 
2179 void TemplateTable::_return(TosState state)
2180 {
2181   transition(state, state);
2182   assert(_desc->calls_vm(),
2183          "inconsistent calls_vm information"); // call in remove_activation
2184 
2185   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2186     assert(state == vtos, "only valid state");
2187 
2188     __ ldr(c_rarg1, aaddress(0));
2189     __ load_klass(r3, c_rarg1);
2190     __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2191     Label skip_register_finalizer;
2192     __ tbz(r3, exact_log2(JVM_ACC_HAS_FINALIZER), skip_register_finalizer);
2193 
2194     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2195 
2196     __ bind(skip_register_finalizer);
2197   }
2198 
2199   // Issue a StoreStore barrier after all stores but before return
2200   // from any constructor for any class with a final field.  We don't
2201   // know if this is a finalizer, so we always do so.
2202   if (_desc->bytecode() == Bytecodes::_return)
2203     __ membar(MacroAssembler::StoreStore);
2204 
2205   // Narrow result if state is itos but result type is smaller.
2206   // Need to narrow in the return bytecode rather than in generate_return_entry
2207   // since compiled code callers expect the result to already be narrowed.
2208   if (state == itos) {
2209     __ narrow(r0);
2210   }
2211 
2212   __ remove_activation(state);
2213   __ ret(lr);
2214 }
2215 
2216 // ----------------------------------------------------------------------------
2217 // Volatile variables demand their effects be made known to all CPU's
2218 // in order.  Store buffers on most chips allow reads & writes to
2219 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2220 // without some kind of memory barrier (i.e., it's not sufficient that
2221 // the interpreter does not reorder volatile references, the hardware
2222 // also must not reorder them).
2223 //
2224 // According to the new Java Memory Model (JMM):
2225 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2226 //     writes act as aquire & release, so:
2227 // (2) A read cannot let unrelated NON-volatile memory refs that
2228 //     happen after the read float up to before the read.  It's OK for
2229 //     non-volatile memory refs that happen before the volatile read to
2230 //     float down below it.
2231 // (3) Similar a volatile write cannot let unrelated NON-volatile
2232 //     memory refs that happen BEFORE the write float down to after the
2233 //     write.  It's OK for non-volatile memory refs that happen after the
2234 //     volatile write to float up before it.
2235 //
2236 // We only put in barriers around volatile refs (they are expensive),
2237 // not _between_ memory refs (that would require us to track the
2238 // flavor of the previous memory refs).  Requirements (2) and (3)
2239 // require some barriers before volatile stores and after volatile
2240 // loads.  These nearly cover requirement (1) but miss the
2241 // volatile-store-volatile-load case.  This final case is placed after
2242 // volatile-stores although it could just as well go before
2243 // volatile-loads.
2244 
2245 void TemplateTable::resolve_cache_and_index(int byte_no,
2246                                             Register Rcache,
2247                                             Register index,
2248                                             size_t index_size) {
2249   const Register temp = r19;
2250   assert_different_registers(Rcache, index, temp);
2251 
2252   Label resolved;
2253 
2254   Bytecodes::Code code = bytecode();
2255   switch (code) {
2256   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2257   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2258   }
2259 
2260   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2261   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2262   __ cmp(temp, (int) code);  // have we resolved this bytecode?
2263   __ br(Assembler::EQ, resolved);
2264 
2265   // resolve first time through
2266   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2267   __ mov(temp, (int) code);
2268   __ call_VM(noreg, entry, temp);
2269 
2270   // Update registers with resolved info
2271   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2272   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2273   // so all clients ofthis method must be modified accordingly
2274   __ bind(resolved);
2275 }
2276 
2277 // The Rcache and index registers must be set before call
2278 // n.b unlike x86 cache already includes the index offset
2279 void TemplateTable::load_field_cp_cache_entry(Register obj,
2280                                               Register cache,
2281                                               Register index,
2282                                               Register off,
2283                                               Register flags,
2284                                               bool is_static = false) {
2285   assert_different_registers(cache, index, flags, off);
2286 
2287   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2288   // Field offset
2289   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2290                                           ConstantPoolCacheEntry::f2_offset())));
2291   // Flags
2292   __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2293                                            ConstantPoolCacheEntry::flags_offset())));
2294 
2295   // klass overwrite register
2296   if (is_static) {
2297     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2298                                         ConstantPoolCacheEntry::f1_offset())));
2299     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2300     __ ldr(obj, Address(obj, mirror_offset));
2301   }
2302 }
2303 
2304 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2305                                                Register method,
2306                                                Register itable_index,
2307                                                Register flags,
2308                                                bool is_invokevirtual,
2309                                                bool is_invokevfinal, /*unused*/
2310                                                bool is_invokedynamic) {
2311   // setup registers
2312   const Register cache = rscratch2;
2313   const Register index = r4;
2314   assert_different_registers(method, flags);
2315   assert_different_registers(method, cache, index);
2316   assert_different_registers(itable_index, flags);
2317   assert_different_registers(itable_index, cache, index);
2318   // determine constant pool cache field offsets
2319   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2320   const int method_offset = in_bytes(
2321     ConstantPoolCache::base_offset() +
2322       (is_invokevirtual
2323        ? ConstantPoolCacheEntry::f2_offset()
2324        : ConstantPoolCacheEntry::f1_offset()));
2325   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2326                                     ConstantPoolCacheEntry::flags_offset());
2327   // access constant pool cache fields
2328   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2329                                     ConstantPoolCacheEntry::f2_offset());
2330 
2331   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2332   resolve_cache_and_index(byte_no, cache, index, index_size);
2333   __ ldr(method, Address(cache, method_offset));
2334 
2335   if (itable_index != noreg) {
2336     __ ldr(itable_index, Address(cache, index_offset));
2337   }
2338   __ ldrw(flags, Address(cache, flags_offset));
2339 }
2340 
2341 
2342 // The registers cache and index expected to be set before call.
2343 // Correct values of the cache and index registers are preserved.
2344 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2345                                             bool is_static, bool has_tos) {
2346   // do the JVMTI work here to avoid disturbing the register state below
2347   // We use c_rarg registers here because we want to use the register used in
2348   // the call to the VM
2349   if (JvmtiExport::can_post_field_access()) {
2350     // Check to see if a field access watch has been set before we
2351     // take the time to call into the VM.
2352     Label L1;
2353     assert_different_registers(cache, index, r0);
2354     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2355     __ ldrw(r0, Address(rscratch1));
2356     __ cbzw(r0, L1);
2357 
2358     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2359     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2360 
2361     if (is_static) {
2362       __ mov(c_rarg1, zr); // NULL object reference
2363     } else {
2364       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2365       __ verify_oop(c_rarg1);
2366     }
2367     // c_rarg1: object pointer or NULL
2368     // c_rarg2: cache entry pointer
2369     // c_rarg3: jvalue object on the stack
2370     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2371                                        InterpreterRuntime::post_field_access),
2372                c_rarg1, c_rarg2, c_rarg3);
2373     __ get_cache_and_index_at_bcp(cache, index, 1);
2374     __ bind(L1);
2375   }
2376 }
2377 
2378 void TemplateTable::pop_and_check_object(Register r)
2379 {
2380   __ pop_ptr(r);
2381   __ null_check(r);  // for field access must check obj.
2382   __ verify_oop(r);
2383 }
2384 
2385 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2386 {
2387   const Register cache = r2;
2388   const Register index = r3;
2389   const Register obj   = r4;
2390   const Register off   = r19;
2391   const Register flags = r0;
2392   const Register bc    = r4; // uses same reg as obj, so don't mix them
2393 
2394   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2395   jvmti_post_field_access(cache, index, is_static, false);
2396   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2397 
2398   if (!is_static) {
2399     // obj is on the stack
2400     pop_and_check_object(obj);
2401   }
2402 
2403   const Address field(obj, off);
2404 
2405   Label Done, notByte, notBool, notInt, notShort, notChar,
2406               notLong, notFloat, notObj, notDouble;
2407 
2408   // x86 uses a shift and mask or wings it with a shift plus assert
2409   // the mask is not needed. aarch64 just uses bitfield extract
2410   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2411 
2412   assert(btos == 0, "change code, btos != 0");
2413   __ cbnz(flags, notByte);
2414 
2415   // Don't rewrite getstatic, only getfield
2416   if (is_static) rc = may_not_rewrite;
2417 
2418   // btos
2419   __ load_signed_byte(r0, field);
2420   __ push(btos);
2421   // Rewrite bytecode to be faster
2422   if (rc == may_rewrite) {
2423     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2424   }
2425   __ b(Done);
2426 
2427   __ bind(notByte);
2428   __ cmp(flags, ztos);
2429   __ br(Assembler::NE, notBool);
2430 
2431   // ztos (same code as btos)
2432   __ ldrsb(r0, field);
2433   __ push(ztos);
2434   // Rewrite bytecode to be faster
2435   if (rc == may_rewrite) {
2436     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2437     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2438   }
2439   __ b(Done);
2440 
2441   __ bind(notBool);
2442   __ cmp(flags, atos);
2443   __ br(Assembler::NE, notObj);
2444   // atos
2445   __ load_heap_oop(r0, field);
2446   __ push(atos);
2447   if (rc == may_rewrite) {
2448     patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2449   }
2450   __ b(Done);
2451 
2452   __ bind(notObj);
2453   __ cmp(flags, itos);
2454   __ br(Assembler::NE, notInt);
2455   // itos
2456   __ ldrw(r0, field);
2457   __ push(itos);
2458   // Rewrite bytecode to be faster
2459   if (rc == may_rewrite) {
2460     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2461   }
2462   __ b(Done);
2463 
2464   __ bind(notInt);
2465   __ cmp(flags, ctos);
2466   __ br(Assembler::NE, notChar);
2467   // ctos
2468   __ load_unsigned_short(r0, field);
2469   __ push(ctos);
2470   // Rewrite bytecode to be faster
2471   if (rc == may_rewrite) {
2472     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2473   }
2474   __ b(Done);
2475 
2476   __ bind(notChar);
2477   __ cmp(flags, stos);
2478   __ br(Assembler::NE, notShort);
2479   // stos
2480   __ load_signed_short(r0, field);
2481   __ push(stos);
2482   // Rewrite bytecode to be faster
2483   if (rc == may_rewrite) {
2484     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2485   }
2486   __ b(Done);
2487 
2488   __ bind(notShort);
2489   __ cmp(flags, ltos);
2490   __ br(Assembler::NE, notLong);
2491   // ltos
2492   __ ldr(r0, field);
2493   __ push(ltos);
2494   // Rewrite bytecode to be faster
2495   if (rc == may_rewrite) {
2496     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2497   }
2498   __ b(Done);
2499 
2500   __ bind(notLong);
2501   __ cmp(flags, ftos);
2502   __ br(Assembler::NE, notFloat);
2503   // ftos
2504   __ ldrs(v0, field);
2505   __ push(ftos);
2506   // Rewrite bytecode to be faster
2507   if (rc == may_rewrite) {
2508     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2509   }
2510   __ b(Done);
2511 
2512   __ bind(notFloat);
2513 #ifdef ASSERT
2514   __ cmp(flags, dtos);
2515   __ br(Assembler::NE, notDouble);
2516 #endif
2517   // dtos
2518   __ ldrd(v0, field);
2519   __ push(dtos);
2520   // Rewrite bytecode to be faster
2521   if (rc == may_rewrite) {
2522     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2523   }
2524 #ifdef ASSERT
2525   __ b(Done);
2526 
2527   __ bind(notDouble);
2528   __ stop("Bad state");
2529 #endif
2530 
2531   __ bind(Done);
2532   // It's really not worth bothering to check whether this field
2533   // really is volatile in the slow case.
2534   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2535 }
2536 
2537 
2538 void TemplateTable::getfield(int byte_no)
2539 {
2540   getfield_or_static(byte_no, false);
2541 }
2542 
2543 void TemplateTable::nofast_getfield(int byte_no) {
2544   getfield_or_static(byte_no, false, may_not_rewrite);
2545 }
2546 
2547 void TemplateTable::getstatic(int byte_no)
2548 {
2549   getfield_or_static(byte_no, true);
2550 }
2551 
2552 // The registers cache and index expected to be set before call.
2553 // The function may destroy various registers, just not the cache and index registers.
2554 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2555   transition(vtos, vtos);
2556 
2557   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2558 
2559   if (JvmtiExport::can_post_field_modification()) {
2560     // Check to see if a field modification watch has been set before
2561     // we take the time to call into the VM.
2562     Label L1;
2563     assert_different_registers(cache, index, r0);
2564     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2565     __ ldrw(r0, Address(rscratch1));
2566     __ cbz(r0, L1);
2567 
2568     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2569 
2570     if (is_static) {
2571       // Life is simple.  Null out the object pointer.
2572       __ mov(c_rarg1, zr);
2573     } else {
2574       // Life is harder. The stack holds the value on top, followed by
2575       // the object.  We don't know the size of the value, though; it
2576       // could be one or two words depending on its type. As a result,
2577       // we must find the type to determine where the object is.
2578       __ ldrw(c_rarg3, Address(c_rarg2,
2579                                in_bytes(cp_base_offset +
2580                                         ConstantPoolCacheEntry::flags_offset())));
2581       __ lsr(c_rarg3, c_rarg3,
2582              ConstantPoolCacheEntry::tos_state_shift);
2583       ConstantPoolCacheEntry::verify_tos_state_shift();
2584       Label nope2, done, ok;
2585       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2586       __ cmpw(c_rarg3, ltos);
2587       __ br(Assembler::EQ, ok);
2588       __ cmpw(c_rarg3, dtos);
2589       __ br(Assembler::NE, nope2);
2590       __ bind(ok);
2591       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2592       __ bind(nope2);
2593     }
2594     // cache entry pointer
2595     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2596     // object (tos)
2597     __ mov(c_rarg3, esp);
2598     // c_rarg1: object pointer set up above (NULL if static)
2599     // c_rarg2: cache entry pointer
2600     // c_rarg3: jvalue object on the stack
2601     __ call_VM(noreg,
2602                CAST_FROM_FN_PTR(address,
2603                                 InterpreterRuntime::post_field_modification),
2604                c_rarg1, c_rarg2, c_rarg3);
2605     __ get_cache_and_index_at_bcp(cache, index, 1);
2606     __ bind(L1);
2607   }
2608 }
2609 
2610 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2611   transition(vtos, vtos);
2612 
2613   const Register cache = r2;
2614   const Register index = r3;
2615   const Register obj   = r2;
2616   const Register off   = r19;
2617   const Register flags = r0;
2618   const Register bc    = r4;
2619 
2620   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2621   jvmti_post_field_mod(cache, index, is_static);
2622   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2623 
2624   Label Done;
2625   __ mov(r5, flags);
2626 
2627   {
2628     Label notVolatile;
2629     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2630     __ membar(MacroAssembler::StoreStore);
2631     __ bind(notVolatile);
2632   }
2633 
2634   // field address
2635   const Address field(obj, off);
2636 
2637   Label notByte, notBool, notInt, notShort, notChar,
2638         notLong, notFloat, notObj, notDouble;
2639 
2640   // x86 uses a shift and mask or wings it with a shift plus assert
2641   // the mask is not needed. aarch64 just uses bitfield extract
2642   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2643 
2644   assert(btos == 0, "change code, btos != 0");
2645   __ cbnz(flags, notByte);
2646 
2647   // Don't rewrite putstatic, only putfield
2648   if (is_static) rc = may_not_rewrite;
2649 
2650   // btos
2651   {
2652     __ pop(btos);
2653     if (!is_static) pop_and_check_object(obj);
2654     __ strb(r0, field);
2655     if (rc == may_rewrite) {
2656       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2657     }
2658     __ b(Done);
2659   }
2660 
2661   __ bind(notByte);
2662   __ cmp(flags, ztos);
2663   __ br(Assembler::NE, notBool);
2664 
2665   // ztos
2666   {
2667     __ pop(ztos);
2668     if (!is_static) pop_and_check_object(obj);
2669     __ andw(r0, r0, 0x1);
2670     __ strb(r0, field);
2671     if (rc == may_rewrite) {
2672       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2673     }
2674     __ b(Done);
2675   }
2676 
2677   __ bind(notBool);
2678   __ cmp(flags, atos);
2679   __ br(Assembler::NE, notObj);
2680 
2681   // atos
2682   {
2683     __ pop(atos);
2684     if (!is_static) pop_and_check_object(obj);
2685     // Store into the field
2686     do_oop_store(_masm, field, r0, _bs->kind(), false);
2687     if (rc == may_rewrite) {
2688       patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2689     }
2690     __ b(Done);
2691   }
2692 
2693   __ bind(notObj);
2694   __ cmp(flags, itos);
2695   __ br(Assembler::NE, notInt);
2696 
2697   // itos
2698   {
2699     __ pop(itos);
2700     if (!is_static) pop_and_check_object(obj);
2701     __ strw(r0, field);
2702     if (rc == may_rewrite) {
2703       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
2704     }
2705     __ b(Done);
2706   }
2707 
2708   __ bind(notInt);
2709   __ cmp(flags, ctos);
2710   __ br(Assembler::NE, notChar);
2711 
2712   // ctos
2713   {
2714     __ pop(ctos);
2715     if (!is_static) pop_and_check_object(obj);
2716     __ strh(r0, field);
2717     if (rc == may_rewrite) {
2718       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
2719     }
2720     __ b(Done);
2721   }
2722 
2723   __ bind(notChar);
2724   __ cmp(flags, stos);
2725   __ br(Assembler::NE, notShort);
2726 
2727   // stos
2728   {
2729     __ pop(stos);
2730     if (!is_static) pop_and_check_object(obj);
2731     __ strh(r0, field);
2732     if (rc == may_rewrite) {
2733       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
2734     }
2735     __ b(Done);
2736   }
2737 
2738   __ bind(notShort);
2739   __ cmp(flags, ltos);
2740   __ br(Assembler::NE, notLong);
2741 
2742   // ltos
2743   {
2744     __ pop(ltos);
2745     if (!is_static) pop_and_check_object(obj);
2746     __ str(r0, field);
2747     if (rc == may_rewrite) {
2748       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
2749     }
2750     __ b(Done);
2751   }
2752 
2753   __ bind(notLong);
2754   __ cmp(flags, ftos);
2755   __ br(Assembler::NE, notFloat);
2756 
2757   // ftos
2758   {
2759     __ pop(ftos);
2760     if (!is_static) pop_and_check_object(obj);
2761     __ strs(v0, field);
2762     if (rc == may_rewrite) {
2763       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
2764     }
2765     __ b(Done);
2766   }
2767 
2768   __ bind(notFloat);
2769 #ifdef ASSERT
2770   __ cmp(flags, dtos);
2771   __ br(Assembler::NE, notDouble);
2772 #endif
2773 
2774   // dtos
2775   {
2776     __ pop(dtos);
2777     if (!is_static) pop_and_check_object(obj);
2778     __ strd(v0, field);
2779     if (rc == may_rewrite) {
2780       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
2781     }
2782   }
2783 
2784 #ifdef ASSERT
2785   __ b(Done);
2786 
2787   __ bind(notDouble);
2788   __ stop("Bad state");
2789 #endif
2790 
2791   __ bind(Done);
2792 
2793   {
2794     Label notVolatile;
2795     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2796     __ membar(MacroAssembler::StoreLoad);
2797     __ bind(notVolatile);
2798   }
2799 }
2800 
2801 void TemplateTable::putfield(int byte_no)
2802 {
2803   putfield_or_static(byte_no, false);
2804 }
2805 
2806 void TemplateTable::nofast_putfield(int byte_no) {
2807   putfield_or_static(byte_no, false, may_not_rewrite);
2808 }
2809 
2810 void TemplateTable::putstatic(int byte_no) {
2811   putfield_or_static(byte_no, true);
2812 }
2813 
2814 void TemplateTable::jvmti_post_fast_field_mod()
2815 {
2816   if (JvmtiExport::can_post_field_modification()) {
2817     // Check to see if a field modification watch has been set before
2818     // we take the time to call into the VM.
2819     Label L2;
2820     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2821     __ ldrw(c_rarg3, Address(rscratch1));
2822     __ cbzw(c_rarg3, L2);
2823     __ pop_ptr(r19);                  // copy the object pointer from tos
2824     __ verify_oop(r19);
2825     __ push_ptr(r19);                 // put the object pointer back on tos
2826     // Save tos values before call_VM() clobbers them. Since we have
2827     // to do it for every data type, we use the saved values as the
2828     // jvalue object.
2829     switch (bytecode()) {          // load values into the jvalue object
2830     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
2831     case Bytecodes::_fast_bputfield: // fall through
2832     case Bytecodes::_fast_zputfield: // fall through
2833     case Bytecodes::_fast_sputfield: // fall through
2834     case Bytecodes::_fast_cputfield: // fall through
2835     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
2836     case Bytecodes::_fast_dputfield: __ push_d(); break;
2837     case Bytecodes::_fast_fputfield: __ push_f(); break;
2838     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
2839 
2840     default:
2841       ShouldNotReachHere();
2842     }
2843     __ mov(c_rarg3, esp);             // points to jvalue on the stack
2844     // access constant pool cache entry
2845     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
2846     __ verify_oop(r19);
2847     // r19: object pointer copied above
2848     // c_rarg2: cache entry pointer
2849     // c_rarg3: jvalue object on the stack
2850     __ call_VM(noreg,
2851                CAST_FROM_FN_PTR(address,
2852                                 InterpreterRuntime::post_field_modification),
2853                r19, c_rarg2, c_rarg3);
2854 
2855     switch (bytecode()) {             // restore tos values
2856     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
2857     case Bytecodes::_fast_bputfield: // fall through
2858     case Bytecodes::_fast_zputfield: // fall through
2859     case Bytecodes::_fast_sputfield: // fall through
2860     case Bytecodes::_fast_cputfield: // fall through
2861     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
2862     case Bytecodes::_fast_dputfield: __ pop_d(); break;
2863     case Bytecodes::_fast_fputfield: __ pop_f(); break;
2864     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
2865     }
2866     __ bind(L2);
2867   }
2868 }
2869 
2870 void TemplateTable::fast_storefield(TosState state)
2871 {
2872   transition(state, vtos);
2873 
2874   ByteSize base = ConstantPoolCache::base_offset();
2875 
2876   jvmti_post_fast_field_mod();
2877 
2878   // access constant pool cache
2879   __ get_cache_and_index_at_bcp(r2, r1, 1);
2880 
2881   // test for volatile with r3
2882   __ ldrw(r3, Address(r2, in_bytes(base +
2883                                    ConstantPoolCacheEntry::flags_offset())));
2884 
2885   // replace index with field offset from cache entry
2886   __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
2887 
2888   {
2889     Label notVolatile;
2890     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2891     __ membar(MacroAssembler::StoreStore);
2892     __ bind(notVolatile);
2893   }
2894 
2895   Label notVolatile;
2896 
2897   // Get object from stack
2898   pop_and_check_object(r2);
2899 
2900   // field address
2901   const Address field(r2, r1);
2902 
2903   // access field
2904   switch (bytecode()) {
2905   case Bytecodes::_fast_aputfield:
2906     do_oop_store(_masm, field, r0, _bs->kind(), false);
2907     break;
2908   case Bytecodes::_fast_lputfield:
2909     __ str(r0, field);
2910     break;
2911   case Bytecodes::_fast_iputfield:
2912     __ strw(r0, field);
2913     break;
2914   case Bytecodes::_fast_zputfield:
2915     __ andw(r0, r0, 0x1);  // boolean is true if LSB is 1
2916     // fall through to bputfield
2917   case Bytecodes::_fast_bputfield:
2918     __ strb(r0, field);
2919     break;
2920   case Bytecodes::_fast_sputfield:
2921     // fall through
2922   case Bytecodes::_fast_cputfield:
2923     __ strh(r0, field);
2924     break;
2925   case Bytecodes::_fast_fputfield:
2926     __ strs(v0, field);
2927     break;
2928   case Bytecodes::_fast_dputfield:
2929     __ strd(v0, field);
2930     break;
2931   default:
2932     ShouldNotReachHere();
2933   }
2934 
2935   {
2936     Label notVolatile;
2937     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2938     __ membar(MacroAssembler::StoreLoad);
2939     __ bind(notVolatile);
2940   }
2941 }
2942 
2943 
2944 void TemplateTable::fast_accessfield(TosState state)
2945 {
2946   transition(atos, state);
2947   // Do the JVMTI work here to avoid disturbing the register state below
2948   if (JvmtiExport::can_post_field_access()) {
2949     // Check to see if a field access watch has been set before we
2950     // take the time to call into the VM.
2951     Label L1;
2952     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2953     __ ldrw(r2, Address(rscratch1));
2954     __ cbzw(r2, L1);
2955     // access constant pool cache entry
2956     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
2957     __ verify_oop(r0);
2958     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
2959     __ mov(c_rarg1, r0);
2960     // c_rarg1: object pointer copied above
2961     // c_rarg2: cache entry pointer
2962     __ call_VM(noreg,
2963                CAST_FROM_FN_PTR(address,
2964                                 InterpreterRuntime::post_field_access),
2965                c_rarg1, c_rarg2);
2966     __ pop_ptr(r0); // restore object pointer
2967     __ bind(L1);
2968   }
2969 
2970   // access constant pool cache
2971   __ get_cache_and_index_at_bcp(r2, r1, 1);
2972   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
2973                                   ConstantPoolCacheEntry::f2_offset())));
2974   __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
2975                                    ConstantPoolCacheEntry::flags_offset())));
2976 
2977   // r0: object
2978   __ verify_oop(r0);
2979   __ null_check(r0);
2980   const Address field(r0, r1);
2981 
2982   // access field
2983   switch (bytecode()) {
2984   case Bytecodes::_fast_agetfield:
2985     __ load_heap_oop(r0, field);
2986     __ verify_oop(r0);
2987     break;
2988   case Bytecodes::_fast_lgetfield:
2989     __ ldr(r0, field);
2990     break;
2991   case Bytecodes::_fast_igetfield:
2992     __ ldrw(r0, field);
2993     break;
2994   case Bytecodes::_fast_bgetfield:
2995     __ load_signed_byte(r0, field);
2996     break;
2997   case Bytecodes::_fast_sgetfield:
2998     __ load_signed_short(r0, field);
2999     break;
3000   case Bytecodes::_fast_cgetfield:
3001     __ load_unsigned_short(r0, field);
3002     break;
3003   case Bytecodes::_fast_fgetfield:
3004     __ ldrs(v0, field);
3005     break;
3006   case Bytecodes::_fast_dgetfield:
3007     __ ldrd(v0, field);
3008     break;
3009   default:
3010     ShouldNotReachHere();
3011   }
3012   {
3013     Label notVolatile;
3014     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3015     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3016     __ bind(notVolatile);
3017   }
3018 }
3019 
3020 void TemplateTable::fast_xaccess(TosState state)
3021 {
3022   transition(vtos, state);
3023 
3024   // get receiver
3025   __ ldr(r0, aaddress(0));
3026   // access constant pool cache
3027   __ get_cache_and_index_at_bcp(r2, r3, 2);
3028   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3029                                   ConstantPoolCacheEntry::f2_offset())));
3030   // make sure exception is reported in correct bcp range (getfield is
3031   // next instruction)
3032   __ increment(rbcp);
3033   __ null_check(r0);
3034   switch (state) {
3035   case itos:
3036     __ ldrw(r0, Address(r0, r1, Address::lsl(0)));
3037     break;
3038   case atos:
3039     __ load_heap_oop(r0, Address(r0, r1, Address::lsl(0)));
3040     __ verify_oop(r0);
3041     break;
3042   case ftos:
3043     __ ldrs(v0, Address(r0, r1, Address::lsl(0)));
3044     break;
3045   default:
3046     ShouldNotReachHere();
3047   }
3048 
3049   {
3050     Label notVolatile;
3051     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3052                                      ConstantPoolCacheEntry::flags_offset())));
3053     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3054     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3055     __ bind(notVolatile);
3056   }
3057 
3058   __ decrement(rbcp);
3059 }
3060 
3061 
3062 
3063 //-----------------------------------------------------------------------------
3064 // Calls
3065 
3066 void TemplateTable::count_calls(Register method, Register temp)
3067 {
3068   __ call_Unimplemented();
3069 }
3070 
3071 void TemplateTable::prepare_invoke(int byte_no,
3072                                    Register method, // linked method (or i-klass)
3073                                    Register index,  // itable index, MethodType, etc.
3074                                    Register recv,   // if caller wants to see it
3075                                    Register flags   // if caller wants to test it
3076                                    ) {
3077   // determine flags
3078   Bytecodes::Code code = bytecode();
3079   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3080   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3081   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3082   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3083   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3084   const bool load_receiver       = (recv  != noreg);
3085   const bool save_flags          = (flags != noreg);
3086   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3087   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3088   assert(flags == noreg || flags == r3, "");
3089   assert(recv  == noreg || recv  == r2, "");
3090 
3091   // setup registers & access constant pool cache
3092   if (recv  == noreg)  recv  = r2;
3093   if (flags == noreg)  flags = r3;
3094   assert_different_registers(method, index, recv, flags);
3095 
3096   // save 'interpreter return address'
3097   __ save_bcp();
3098 
3099   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3100 
3101   // maybe push appendix to arguments (just before return address)
3102   if (is_invokedynamic || is_invokehandle) {
3103     Label L_no_push;
3104     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3105     // Push the appendix as a trailing parameter.
3106     // This must be done before we get the receiver,
3107     // since the parameter_size includes it.
3108     __ push(r19);
3109     __ mov(r19, index);
3110     assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
3111     __ load_resolved_reference_at_index(index, r19);
3112     __ pop(r19);
3113     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3114     __ bind(L_no_push);
3115   }
3116 
3117   // load receiver if needed (note: no return address pushed yet)
3118   if (load_receiver) {
3119     __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3120     // FIXME -- is this actually correct? looks like it should be 2
3121     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3122     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3123     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3124     // __ movptr(recv, recv_addr);
3125     __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3126     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3127     __ verify_oop(recv);
3128   }
3129 
3130   // compute return type
3131   // x86 uses a shift and mask or wings it with a shift plus assert
3132   // the mask is not needed. aarch64 just uses bitfield extract
3133   __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3134   // load return address
3135   {
3136     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3137     __ mov(rscratch1, table_addr);
3138     __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3139   }
3140 }
3141 
3142 
3143 void TemplateTable::invokevirtual_helper(Register index,
3144                                          Register recv,
3145                                          Register flags)
3146 {
3147   // Uses temporary registers r0, r3
3148   assert_different_registers(index, recv, r0, r3);
3149   // Test for an invoke of a final method
3150   Label notFinal;
3151   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3152 
3153   const Register method = index;  // method must be rmethod
3154   assert(method == rmethod,
3155          "methodOop must be rmethod for interpreter calling convention");
3156 
3157   // do the call - the index is actually the method to call
3158   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3159 
3160   // It's final, need a null check here!
3161   __ null_check(recv);
3162 
3163   // profile this call
3164   __ profile_final_call(r0);
3165   __ profile_arguments_type(r0, method, r4, true);
3166 
3167   __ jump_from_interpreted(method, r0);
3168 
3169   __ bind(notFinal);
3170 
3171   // get receiver klass
3172   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3173   __ load_klass(r0, recv);
3174 
3175   // profile this call
3176   __ profile_virtual_call(r0, rlocals, r3);
3177 
3178   // get target methodOop & entry point
3179   __ lookup_virtual_method(r0, index, method);
3180   __ profile_arguments_type(r3, method, r4, true);
3181   // FIXME -- this looks completely redundant. is it?
3182   // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3183   __ jump_from_interpreted(method, r3);
3184 }
3185 
3186 void TemplateTable::invokevirtual(int byte_no)
3187 {
3188   transition(vtos, vtos);
3189   assert(byte_no == f2_byte, "use this argument");
3190 
3191   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3192 
3193   // rmethod: index (actually a Method*)
3194   // r2: receiver
3195   // r3: flags
3196 
3197   invokevirtual_helper(rmethod, r2, r3);
3198 }
3199 
3200 void TemplateTable::invokespecial(int byte_no)
3201 {
3202   transition(vtos, vtos);
3203   assert(byte_no == f1_byte, "use this argument");
3204 
3205   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3206                  r2);  // get receiver also for null check
3207   __ verify_oop(r2);
3208   __ null_check(r2);
3209   // do the call
3210   __ profile_call(r0);
3211   __ profile_arguments_type(r0, rmethod, rbcp, false);
3212   __ jump_from_interpreted(rmethod, r0);
3213 }
3214 
3215 void TemplateTable::invokestatic(int byte_no)
3216 {
3217   transition(vtos, vtos);
3218   assert(byte_no == f1_byte, "use this argument");
3219 
3220   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3221   // do the call
3222   __ profile_call(r0);
3223   __ profile_arguments_type(r0, rmethod, r4, false);
3224   __ jump_from_interpreted(rmethod, r0);
3225 }
3226 
3227 void TemplateTable::fast_invokevfinal(int byte_no)
3228 {
3229   __ call_Unimplemented();
3230 }
3231 
3232 void TemplateTable::invokeinterface(int byte_no) {
3233   transition(vtos, vtos);
3234   assert(byte_no == f1_byte, "use this argument");
3235 
3236   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 itable index
3237                  r2, r3); // recv, flags
3238 
3239   // r0: interface klass (from f1)
3240   // rmethod: itable index (from f2)
3241   // r2: receiver
3242   // r3: flags
3243 
3244   // Special case of invokeinterface called for virtual method of
3245   // java.lang.Object.  See cpCacheOop.cpp for details.
3246   // This code isn't produced by javac, but could be produced by
3247   // another compliant java compiler.
3248   Label notMethod;
3249   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notMethod);
3250 
3251   invokevirtual_helper(rmethod, r2, r3);
3252   __ bind(notMethod);
3253 
3254   // Get receiver klass into r3 - also a null check
3255   __ restore_locals();
3256   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3257   __ load_klass(r3, r2);
3258 
3259   // profile this call
3260   __ profile_virtual_call(r3, r13, r19);
3261 
3262   Label no_such_interface, no_such_method;
3263 
3264   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3265                              r3, r0, rmethod,
3266                              // outputs: method, scan temp. reg
3267                              rmethod, r13,
3268                              no_such_interface);
3269 
3270   // rmethod,: methodOop to call
3271   // r2: receiver
3272   // Check for abstract method error
3273   // Note: This should be done more efficiently via a throw_abstract_method_error
3274   //       interpreter entry point and a conditional jump to it in case of a null
3275   //       method.
3276   __ cbz(rmethod, no_such_method);
3277 
3278   __ profile_arguments_type(r3, rmethod, r13, true);
3279 
3280   // do the call
3281   // r2: receiver
3282   // rmethod,: methodOop
3283   __ jump_from_interpreted(rmethod, r3);
3284   __ should_not_reach_here();
3285 
3286   // exception handling code follows...
3287   // note: must restore interpreter registers to canonical
3288   //       state for exception handling to work correctly!
3289 
3290   __ bind(no_such_method);
3291   // throw exception
3292   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3293   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3294   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
3295   // the call_VM checks for exception, so we should never return here.
3296   __ should_not_reach_here();
3297 
3298   __ bind(no_such_interface);
3299   // throw exception
3300   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3301   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3302   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3303                    InterpreterRuntime::throw_IncompatibleClassChangeError));
3304   // the call_VM checks for exception, so we should never return here.
3305   __ should_not_reach_here();
3306   return;
3307 }
3308 
3309 void TemplateTable::invokehandle(int byte_no) {
3310   transition(vtos, vtos);
3311   assert(byte_no == f1_byte, "use this argument");
3312 
3313   prepare_invoke(byte_no, rmethod, r0, r2);
3314   __ verify_method_ptr(r2);
3315   __ verify_oop(r2);
3316   __ null_check(r2);
3317 
3318   // FIXME: profile the LambdaForm also
3319 
3320   // r13 is safe to use here as a scratch reg because it is about to
3321   // be clobbered by jump_from_interpreted().
3322   __ profile_final_call(r13);
3323   __ profile_arguments_type(r13, rmethod, r4, true);
3324 
3325   __ jump_from_interpreted(rmethod, r0);
3326 }
3327 
3328 void TemplateTable::invokedynamic(int byte_no) {
3329   transition(vtos, vtos);
3330   assert(byte_no == f1_byte, "use this argument");
3331 
3332   prepare_invoke(byte_no, rmethod, r0);
3333 
3334   // r0: CallSite object (from cpool->resolved_references[])
3335   // rmethod: MH.linkToCallSite method (from f2)
3336 
3337   // Note:  r0_callsite is already pushed by prepare_invoke
3338 
3339   // %%% should make a type profile for any invokedynamic that takes a ref argument
3340   // profile this call
3341   __ profile_call(rbcp);
3342   __ profile_arguments_type(r3, rmethod, r13, false);
3343 
3344   __ verify_oop(r0);
3345 
3346   __ jump_from_interpreted(rmethod, r0);
3347 }
3348 
3349 
3350 //-----------------------------------------------------------------------------
3351 // Allocation
3352 
3353 void TemplateTable::_new() {
3354   transition(vtos, atos);
3355 
3356   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3357   Label slow_case;
3358   Label done;
3359   Label initialize_header;
3360   Label initialize_object; // including clearing the fields
3361   Label allocate_shared;
3362 
3363   __ get_cpool_and_tags(r4, r0);
3364   // Make sure the class we're about to instantiate has been resolved.
3365   // This is done before loading InstanceKlass to be consistent with the order
3366   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3367   const int tags_offset = Array<u1>::base_offset_in_bytes();
3368   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3369   __ lea(rscratch1, Address(rscratch1, tags_offset));
3370   __ ldarb(rscratch1, rscratch1);
3371   __ cmp(rscratch1, JVM_CONSTANT_Class);
3372   __ br(Assembler::NE, slow_case);
3373 
3374   // get InstanceKlass
3375   __ lea(r4, Address(r4, r3, Address::lsl(3)));
3376   __ ldr(r4, Address(r4, sizeof(ConstantPool)));
3377 
3378   // make sure klass is initialized & doesn't have finalizer
3379   // make sure klass is fully initialized
3380   __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3381   __ cmp(rscratch1, InstanceKlass::fully_initialized);
3382   __ br(Assembler::NE, slow_case);
3383 
3384   // get instance_size in InstanceKlass (scaled to a count of bytes)
3385   __ ldrw(r3,
3386           Address(r4,
3387                   Klass::layout_helper_offset()));
3388   // test to see if it has a finalizer or is malformed in some way
3389   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3390 
3391   // Allocate the instance
3392   // 1) Try to allocate in the TLAB
3393   // 2) if fail and the object is large allocate in the shared Eden
3394   // 3) if the above fails (or is not applicable), go to a slow case
3395   // (creates a new TLAB, etc.)
3396 
3397   const bool allow_shared_alloc =
3398     Universe::heap()->supports_inline_contig_alloc();
3399 
3400   if (UseTLAB) {
3401     __ tlab_allocate(r0, r3, 0, noreg, r1,
3402                      allow_shared_alloc ? allocate_shared : slow_case);
3403 
3404     if (ZeroTLAB) {
3405       // the fields have been already cleared
3406       __ b(initialize_header);
3407     } else {
3408       // initialize both the header and fields
3409       __ b(initialize_object);
3410     }
3411   }
3412 
3413   // Allocation in the shared Eden, if allowed.
3414   //
3415   // r3: instance size in bytes
3416   if (allow_shared_alloc) {
3417     __ bind(allocate_shared);
3418 
3419     __ eden_allocate(r0, r3, 0, r10, slow_case);
3420     __ incr_allocated_bytes(rthread, r3, 0, rscratch1);
3421   }
3422 
3423   if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
3424     // The object is initialized before the header.  If the object size is
3425     // zero, go directly to the header initialization.
3426     __ bind(initialize_object);
3427     __ sub(r3, r3, sizeof(oopDesc));
3428     __ cbz(r3, initialize_header);
3429 
3430     // Initialize object fields
3431     {
3432       __ add(r2, r0, sizeof(oopDesc));
3433       Label loop;
3434       __ bind(loop);
3435       __ str(zr, Address(__ post(r2, BytesPerLong)));
3436       __ sub(r3, r3, BytesPerLong);
3437       __ cbnz(r3, loop);
3438     }
3439 
3440     // initialize object header only.
3441     __ bind(initialize_header);
3442     if (UseBiasedLocking) {
3443       __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3444     } else {
3445       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3446     }
3447     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3448     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3449     __ store_klass(r0, r4, rscratch1);      // store klass last
3450 
3451     {
3452       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3453       // Trigger dtrace event for fastpath
3454       __ push(atos); // save the return value
3455       __ call_VM_leaf(
3456            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3457       __ pop(atos); // restore the return value
3458 
3459     }
3460     __ b(done);
3461   }
3462 
3463   // slow case
3464   __ bind(slow_case);
3465   __ get_constant_pool(c_rarg1);
3466   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3467   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3468   __ verify_oop(r0);
3469 
3470   // continue
3471   __ bind(done);
3472   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3473   __ membar(Assembler::StoreStore);
3474 }
3475 
3476 void TemplateTable::newarray() {
3477   transition(itos, atos);
3478   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3479   __ mov(c_rarg2, r0);
3480   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3481           c_rarg1, c_rarg2);
3482   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3483   __ membar(Assembler::StoreStore);
3484 }
3485 
3486 void TemplateTable::anewarray() {
3487   transition(itos, atos);
3488   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3489   __ get_constant_pool(c_rarg1);
3490   __ mov(c_rarg3, r0);
3491   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3492           c_rarg1, c_rarg2, c_rarg3);
3493   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3494   __ membar(Assembler::StoreStore);
3495 }
3496 
3497 void TemplateTable::arraylength() {
3498   transition(atos, itos);
3499   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3500   __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3501 }
3502 
3503 void TemplateTable::checkcast()
3504 {
3505   transition(atos, atos);
3506   Label done, is_null, ok_is_subtype, quicked, resolved;
3507   __ cbz(r0, is_null);
3508 
3509   // Get cpool & tags index
3510   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3511   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3512   // See if bytecode has already been quicked
3513   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3514   __ lea(r1, Address(rscratch1, r19));
3515   __ ldarb(r1, r1);
3516   __ cmp(r1, JVM_CONSTANT_Class);
3517   __ br(Assembler::EQ, quicked);
3518 
3519   __ push(atos); // save receiver for result, and for GC
3520   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3521   // vm_result_2 has metadata result
3522   __ get_vm_result_2(r0, rthread);
3523   __ pop(r3); // restore receiver
3524   __ b(resolved);
3525 
3526   // Get superklass in r0 and subklass in r3
3527   __ bind(quicked);
3528   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3529   __ lea(r0, Address(r2, r19, Address::lsl(3)));
3530   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3531 
3532   __ bind(resolved);
3533   __ load_klass(r19, r3);
3534 
3535   // Generate subtype check.  Blows r2, r5.  Object in r3.
3536   // Superklass in r0.  Subklass in r19.
3537   __ gen_subtype_check(r19, ok_is_subtype);
3538 
3539   // Come here on failure
3540   __ push(r3);
3541   // object is at TOS
3542   __ b(Interpreter::_throw_ClassCastException_entry);
3543 
3544   // Come here on success
3545   __ bind(ok_is_subtype);
3546   __ mov(r0, r3); // Restore object in r3
3547 
3548   // Collect counts on whether this test sees NULLs a lot or not.
3549   if (ProfileInterpreter) {
3550     __ b(done);
3551     __ bind(is_null);
3552     __ profile_null_seen(r2);
3553   } else {
3554     __ bind(is_null);   // same as 'done'
3555   }
3556   __ bind(done);
3557 }
3558 
3559 void TemplateTable::instanceof() {
3560   transition(atos, itos);
3561   Label done, is_null, ok_is_subtype, quicked, resolved;
3562   __ cbz(r0, is_null);
3563 
3564   // Get cpool & tags index
3565   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3566   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3567   // See if bytecode has already been quicked
3568   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3569   __ lea(r1, Address(rscratch1, r19));
3570   __ ldarb(r1, r1);
3571   __ cmp(r1, JVM_CONSTANT_Class);
3572   __ br(Assembler::EQ, quicked);
3573 
3574   __ push(atos); // save receiver for result, and for GC
3575   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3576   // vm_result_2 has metadata result
3577   __ get_vm_result_2(r0, rthread);
3578   __ pop(r3); // restore receiver
3579   __ verify_oop(r3);
3580   __ load_klass(r3, r3);
3581   __ b(resolved);
3582 
3583   // Get superklass in r0 and subklass in r3
3584   __ bind(quicked);
3585   __ load_klass(r3, r0);
3586   __ lea(r0, Address(r2, r19, Address::lsl(3)));
3587   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3588 
3589   __ bind(resolved);
3590 
3591   // Generate subtype check.  Blows r2, r5
3592   // Superklass in r0.  Subklass in r3.
3593   __ gen_subtype_check(r3, ok_is_subtype);
3594 
3595   // Come here on failure
3596   __ mov(r0, 0);
3597   __ b(done);
3598   // Come here on success
3599   __ bind(ok_is_subtype);
3600   __ mov(r0, 1);
3601 
3602   // Collect counts on whether this test sees NULLs a lot or not.
3603   if (ProfileInterpreter) {
3604     __ b(done);
3605     __ bind(is_null);
3606     __ profile_null_seen(r2);
3607   } else {
3608     __ bind(is_null);   // same as 'done'
3609   }
3610   __ bind(done);
3611   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
3612   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
3613 }
3614 
3615 //-----------------------------------------------------------------------------
3616 // Breakpoints
3617 void TemplateTable::_breakpoint() {
3618   // Note: We get here even if we are single stepping..
3619   // jbug inists on setting breakpoints at every bytecode
3620   // even if we are in single step mode.
3621 
3622   transition(vtos, vtos);
3623 
3624   // get the unpatched byte code
3625   __ get_method(c_rarg1);
3626   __ call_VM(noreg,
3627              CAST_FROM_FN_PTR(address,
3628                               InterpreterRuntime::get_original_bytecode_at),
3629              c_rarg1, rbcp);
3630   __ mov(r19, r0);
3631 
3632   // post the breakpoint event
3633   __ call_VM(noreg,
3634              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3635              rmethod, rbcp);
3636 
3637   // complete the execution of original bytecode
3638   __ mov(rscratch1, r19);
3639   __ dispatch_only_normal(vtos);
3640 }
3641 
3642 //-----------------------------------------------------------------------------
3643 // Exceptions
3644 
3645 void TemplateTable::athrow() {
3646   transition(atos, vtos);
3647   __ null_check(r0);
3648   __ b(Interpreter::throw_exception_entry());
3649 }
3650 
3651 //-----------------------------------------------------------------------------
3652 // Synchronization
3653 //
3654 // Note: monitorenter & exit are symmetric routines; which is reflected
3655 //       in the assembly code structure as well
3656 //
3657 // Stack layout:
3658 //
3659 // [expressions  ] <--- esp               = expression stack top
3660 // ..
3661 // [expressions  ]
3662 // [monitor entry] <--- monitor block top = expression stack bot
3663 // ..
3664 // [monitor entry]
3665 // [frame data   ] <--- monitor block bot
3666 // ...
3667 // [saved rbp    ] <--- rbp
3668 void TemplateTable::monitorenter()
3669 {
3670   transition(atos, vtos);
3671 
3672   // check for NULL object
3673   __ null_check(r0);
3674 
3675   const Address monitor_block_top(
3676         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3677   const Address monitor_block_bot(
3678         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3679   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3680 
3681   Label allocated;
3682 
3683   // initialize entry pointer
3684   __ mov(c_rarg1, zr); // points to free slot or NULL
3685 
3686   // find a free slot in the monitor block (result in c_rarg1)
3687   {
3688     Label entry, loop, exit;
3689     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
3690                                         // starting with top-most entry
3691     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3692 
3693     __ b(entry);
3694 
3695     __ bind(loop);
3696     // check if current entry is used
3697     // if not used then remember entry in c_rarg1
3698     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
3699     __ cmp(zr, rscratch1);
3700     __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
3701     // check if current entry is for same object
3702     __ cmp(r0, rscratch1);
3703     // if same object then stop searching
3704     __ br(Assembler::EQ, exit);
3705     // otherwise advance to next entry
3706     __ add(c_rarg3, c_rarg3, entry_size);
3707     __ bind(entry);
3708     // check if bottom reached
3709     __ cmp(c_rarg3, c_rarg2);
3710     // if not at bottom then check this entry
3711     __ br(Assembler::NE, loop);
3712     __ bind(exit);
3713   }
3714 
3715   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
3716                             // if found, continue with that on
3717 
3718   // allocate one if there's no free slot
3719   {
3720     Label entry, loop;
3721     // 1. compute new pointers            // rsp: old expression stack top
3722     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
3723     __ sub(esp, esp, entry_size);         // move expression stack top
3724     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3725     __ mov(c_rarg3, esp);                 // set start value for copy loop
3726     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
3727 
3728     __ sub(sp, sp, entry_size);           // make room for the monitor
3729 
3730     __ b(entry);
3731     // 2. move expression stack contents
3732     __ bind(loop);
3733     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3734                                                    // word from old location
3735     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
3736     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
3737     __ bind(entry);
3738     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
3739     __ br(Assembler::NE, loop);      // if not at bottom then
3740                                      // copy next word
3741   }
3742 
3743   // call run-time routine
3744   // c_rarg1: points to monitor entry
3745   __ bind(allocated);
3746 
3747   // Increment bcp to point to the next bytecode, so exception
3748   // handling for async. exceptions work correctly.
3749   // The object has already been poped from the stack, so the
3750   // expression stack looks correct.
3751   __ increment(rbcp);
3752 
3753   // store object
3754   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3755   __ lock_object(c_rarg1);
3756 
3757   // check to make sure this monitor doesn't cause stack overflow after locking
3758   __ save_bcp();  // in case of exception
3759   __ generate_stack_overflow_check(0);
3760 
3761   // The bcp has already been incremented. Just need to dispatch to
3762   // next instruction.
3763   __ dispatch_next(vtos);
3764 }
3765 
3766 
3767 void TemplateTable::monitorexit()
3768 {
3769   transition(atos, vtos);
3770 
3771   // check for NULL object
3772   __ null_check(r0);
3773 
3774   const Address monitor_block_top(
3775         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3776   const Address monitor_block_bot(
3777         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3778   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3779 
3780   Label found;
3781 
3782   // find matching slot
3783   {
3784     Label entry, loop;
3785     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
3786                                         // starting with top-most entry
3787     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3788                                         // of monitor block
3789     __ b(entry);
3790 
3791     __ bind(loop);
3792     // check if current entry is for same object
3793     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3794     __ cmp(r0, rscratch1);
3795     // if same object then stop searching
3796     __ br(Assembler::EQ, found);
3797     // otherwise advance to next entry
3798     __ add(c_rarg1, c_rarg1, entry_size);
3799     __ bind(entry);
3800     // check if bottom reached
3801     __ cmp(c_rarg1, c_rarg2);
3802     // if not at bottom then check this entry
3803     __ br(Assembler::NE, loop);
3804   }
3805 
3806   // error handling. Unlocking was not block-structured
3807   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3808                    InterpreterRuntime::throw_illegal_monitor_state_exception));
3809   __ should_not_reach_here();
3810 
3811   // call run-time routine
3812   __ bind(found);
3813   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
3814   __ unlock_object(c_rarg1);
3815   __ pop_ptr(r0); // discard object
3816 }
3817 
3818 
3819 // Wide instructions
3820 void TemplateTable::wide()
3821 {
3822   __ load_unsigned_byte(r19, at_bcp(1));
3823   __ mov(rscratch1, (address)Interpreter::_wentry_point);
3824   __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
3825   __ br(rscratch1);
3826 }
3827 
3828 
3829 // Multi arrays
3830 void TemplateTable::multianewarray() {
3831   transition(vtos, atos);
3832   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
3833   // last dim is on top of stack; we want address of first one:
3834   // first_addr = last_addr + (ndims - 1) * wordSize
3835   __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
3836   __ sub(c_rarg1, c_rarg1, wordSize);
3837   call_VM(r0,
3838           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3839           c_rarg1);
3840   __ load_unsigned_byte(r1, at_bcp(3));
3841   __ lea(esp, Address(esp, r1, Address::uxtw(3)));
3842 }