1 /*
   2  * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.hpp"
  28 #include "gc/shared/barrierSetAssembler.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/interp_masm.hpp"
  32 #include "interpreter/templateTable.hpp"
  33 #include "memory/universe.hpp"
  34 #include "oops/methodData.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/objArrayKlass.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/frame.inline.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "runtime/stubRoutines.hpp"
  42 #include "runtime/synchronizer.hpp"
  43 
  44 #define __ _masm->
  45 
  46 // Platform-dependent initialization
  47 
  48 void TemplateTable::pd_initialize() {
  49   // No aarch64 specific initialization
  50 }
  51 
  52 // Address computation: local variables
  53 
  54 static inline Address iaddress(int n) {
  55   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  56 }
  57 
  58 static inline Address laddress(int n) {
  59   return iaddress(n + 1);
  60 }
  61 
  62 static inline Address faddress(int n) {
  63   return iaddress(n);
  64 }
  65 
  66 static inline Address daddress(int n) {
  67   return laddress(n);
  68 }
  69 
  70 static inline Address aaddress(int n) {
  71   return iaddress(n);
  72 }
  73 
  74 static inline Address iaddress(Register r) {
  75   return Address(rlocals, r, Address::lsl(3));
  76 }
  77 
  78 static inline Address laddress(Register r, Register scratch,
  79                                InterpreterMacroAssembler* _masm) {
  80   __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
  81   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  82 }
  83 
  84 static inline Address faddress(Register r) {
  85   return iaddress(r);
  86 }
  87 
  88 static inline Address daddress(Register r, Register scratch,
  89                                InterpreterMacroAssembler* _masm) {
  90   return laddress(r, scratch, _masm);
  91 }
  92 
  93 static inline Address aaddress(Register r) {
  94   return iaddress(r);
  95 }
  96 
  97 static inline Address at_rsp() {
  98   return Address(esp, 0);
  99 }
 100 
 101 // At top of Java expression stack which may be different than esp().  It
 102 // isn't for category 1 objects.
 103 static inline Address at_tos   () {
 104   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 105 }
 106 
 107 static inline Address at_tos_p1() {
 108   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 109 }
 110 
 111 static inline Address at_tos_p2() {
 112   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 113 }
 114 
 115 static inline Address at_tos_p3() {
 116   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 117 }
 118 
 119 static inline Address at_tos_p4() {
 120   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 121 }
 122 
 123 static inline Address at_tos_p5() {
 124   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 125 }
 126 
 127 // Condition conversion
 128 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 129   switch (cc) {
 130   case TemplateTable::equal        : return Assembler::NE;
 131   case TemplateTable::not_equal    : return Assembler::EQ;
 132   case TemplateTable::less         : return Assembler::GE;
 133   case TemplateTable::less_equal   : return Assembler::GT;
 134   case TemplateTable::greater      : return Assembler::LE;
 135   case TemplateTable::greater_equal: return Assembler::LT;
 136   }
 137   ShouldNotReachHere();
 138   return Assembler::EQ;
 139 }
 140 
 141 
 142 // Miscelaneous helper routines
 143 // Store an oop (or NULL) at the Address described by obj.
 144 // If val == noreg this means store a NULL
 145 static void do_oop_store(InterpreterMacroAssembler* _masm,
 146                          Address dst,
 147                          Register val,
 148                          DecoratorSet decorators) {
 149   assert(val == noreg || val == r0, "parameter is just for looks");
 150   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 151   bs->store_at(_masm, decorators, T_OBJECT, dst, val, /*tmp1*/ r10, /*tmp2*/ r1);
 152 }
 153 
 154 static void do_oop_load(InterpreterMacroAssembler* _masm,
 155                         Address src,
 156                         Register dst,
 157                         DecoratorSet decorators) {
 158   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 159   bs->load_at(_masm, decorators, T_OBJECT, dst, src, /*tmp1*/ r10, /*tmp_thread*/ r1);
 160 }
 161 
 162 Address TemplateTable::at_bcp(int offset) {
 163   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 164   return Address(rbcp, offset);
 165 }
 166 
 167 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 168                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 169                                    int byte_no)
 170 {
 171   if (!RewriteBytecodes)  return;
 172   Label L_patch_done;
 173 
 174   switch (bc) {
 175   case Bytecodes::_fast_aputfield:
 176   case Bytecodes::_fast_bputfield:
 177   case Bytecodes::_fast_zputfield:
 178   case Bytecodes::_fast_cputfield:
 179   case Bytecodes::_fast_dputfield:
 180   case Bytecodes::_fast_fputfield:
 181   case Bytecodes::_fast_iputfield:
 182   case Bytecodes::_fast_lputfield:
 183   case Bytecodes::_fast_sputfield:
 184     {
 185       // We skip bytecode quickening for putfield instructions when
 186       // the put_code written to the constant pool cache is zero.
 187       // This is required so that every execution of this instruction
 188       // calls out to InterpreterRuntime::resolve_get_put to do
 189       // additional, required work.
 190       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 191       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 192       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 193       __ movw(bc_reg, bc);
 194       __ cbzw(temp_reg, L_patch_done);  // don't patch
 195     }
 196     break;
 197   default:
 198     assert(byte_no == -1, "sanity");
 199     // the pair bytecodes have already done the load.
 200     if (load_bc_into_bc_reg) {
 201       __ movw(bc_reg, bc);
 202     }
 203   }
 204 
 205   if (JvmtiExport::can_post_breakpoint()) {
 206     Label L_fast_patch;
 207     // if a breakpoint is present we can't rewrite the stream directly
 208     __ load_unsigned_byte(temp_reg, at_bcp(0));
 209     __ cmpw(temp_reg, Bytecodes::_breakpoint);
 210     __ br(Assembler::NE, L_fast_patch);
 211     // Let breakpoint table handling rewrite to quicker bytecode
 212     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 213     __ b(L_patch_done);
 214     __ bind(L_fast_patch);
 215   }
 216 
 217 #ifdef ASSERT
 218   Label L_okay;
 219   __ load_unsigned_byte(temp_reg, at_bcp(0));
 220   __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
 221   __ br(Assembler::EQ, L_okay);
 222   __ cmpw(temp_reg, bc_reg);
 223   __ br(Assembler::EQ, L_okay);
 224   __ stop("patching the wrong bytecode");
 225   __ bind(L_okay);
 226 #endif
 227 
 228   // patch bytecode
 229   __ strb(bc_reg, at_bcp(0));
 230   __ bind(L_patch_done);
 231 }
 232 
 233 
 234 // Individual instructions
 235 
 236 void TemplateTable::nop() {
 237   transition(vtos, vtos);
 238   // nothing to do
 239 }
 240 
 241 void TemplateTable::shouldnotreachhere() {
 242   transition(vtos, vtos);
 243   __ stop("shouldnotreachhere bytecode");
 244 }
 245 
 246 void TemplateTable::aconst_null()
 247 {
 248   transition(vtos, atos);
 249   __ mov(r0, 0);
 250 }
 251 
 252 void TemplateTable::iconst(int value)
 253 {
 254   transition(vtos, itos);
 255   __ mov(r0, value);
 256 }
 257 
 258 void TemplateTable::lconst(int value)
 259 {
 260   __ mov(r0, value);
 261 }
 262 
 263 void TemplateTable::fconst(int value)
 264 {
 265   transition(vtos, ftos);
 266   switch (value) {
 267   case 0:
 268     __ fmovs(v0, zr);
 269     break;
 270   case 1:
 271     __ fmovs(v0, 1.0);
 272     break;
 273   case 2:
 274     __ fmovs(v0, 2.0);
 275     break;
 276   default:
 277     ShouldNotReachHere();
 278     break;
 279   }
 280 }
 281 
 282 void TemplateTable::dconst(int value)
 283 {
 284   transition(vtos, dtos);
 285   switch (value) {
 286   case 0:
 287     __ fmovd(v0, zr);
 288     break;
 289   case 1:
 290     __ fmovd(v0, 1.0);
 291     break;
 292   case 2:
 293     __ fmovd(v0, 2.0);
 294     break;
 295   default:
 296     ShouldNotReachHere();
 297     break;
 298   }
 299 }
 300 
 301 void TemplateTable::bipush()
 302 {
 303   transition(vtos, itos);
 304   __ load_signed_byte32(r0, at_bcp(1));
 305 }
 306 
 307 void TemplateTable::sipush()
 308 {
 309   transition(vtos, itos);
 310   __ load_unsigned_short(r0, at_bcp(1));
 311   __ revw(r0, r0);
 312   __ asrw(r0, r0, 16);
 313 }
 314 
 315 void TemplateTable::ldc(bool wide)
 316 {
 317   transition(vtos, vtos);
 318   Label call_ldc, notFloat, notClass, notInt, Done;
 319 
 320   if (wide) {
 321     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 322   } else {
 323     __ load_unsigned_byte(r1, at_bcp(1));
 324   }
 325   __ get_cpool_and_tags(r2, r0);
 326 
 327   const int base_offset = ConstantPool::header_size() * wordSize;
 328   const int tags_offset = Array<u1>::base_offset_in_bytes();
 329 
 330   // get type
 331   __ add(r3, r1, tags_offset);
 332   __ lea(r3, Address(r0, r3));
 333   __ ldarb(r3, r3);
 334 
 335   // unresolved class - get the resolved class
 336   __ cmp(r3, JVM_CONSTANT_UnresolvedClass);
 337   __ br(Assembler::EQ, call_ldc);
 338 
 339   // unresolved class in error state - call into runtime to throw the error
 340   // from the first resolution attempt
 341   __ cmp(r3, JVM_CONSTANT_UnresolvedClassInError);
 342   __ br(Assembler::EQ, call_ldc);
 343 
 344   // resolved class - need to call vm to get java mirror of the class
 345   __ cmp(r3, JVM_CONSTANT_Class);
 346   __ br(Assembler::NE, notClass);
 347 
 348   __ bind(call_ldc);
 349   __ mov(c_rarg1, wide);
 350   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 351   __ push_ptr(r0);
 352   __ verify_oop(r0);
 353   __ b(Done);
 354 
 355   __ bind(notClass);
 356   __ cmp(r3, JVM_CONSTANT_Float);
 357   __ br(Assembler::NE, notFloat);
 358   // ftos
 359   __ adds(r1, r2, r1, Assembler::LSL, 3);
 360   __ ldrs(v0, Address(r1, base_offset));
 361   __ push_f();
 362   __ b(Done);
 363 
 364   __ bind(notFloat);
 365 
 366   __ cmp(r3, JVM_CONSTANT_Integer);
 367   __ br(Assembler::NE, notInt);
 368 
 369   // itos
 370   __ adds(r1, r2, r1, Assembler::LSL, 3);
 371   __ ldrw(r0, Address(r1, base_offset));
 372   __ push_i(r0);
 373   __ b(Done);
 374 
 375   __ bind(notInt);
 376   condy_helper(Done);
 377 
 378   __ bind(Done);
 379 }
 380 
 381 // Fast path for caching oop constants.
 382 void TemplateTable::fast_aldc(bool wide)
 383 {
 384   transition(vtos, atos);
 385 
 386   Register result = r0;
 387   Register tmp = r1;
 388   Register rarg = r2;
 389 
 390   int index_size = wide ? sizeof(u2) : sizeof(u1);
 391 
 392   Label resolved;
 393 
 394   // We are resolved if the resolved reference cache entry contains a
 395   // non-null object (String, MethodType, etc.)
 396   assert_different_registers(result, tmp);
 397   __ get_cache_index_at_bcp(tmp, 1, index_size);
 398   __ load_resolved_reference_at_index(result, tmp);
 399   __ cbnz(result, resolved);
 400 
 401   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 402 
 403   // first time invocation - must resolve first
 404   __ mov(rarg, (int)bytecode());
 405   __ call_VM(result, entry, rarg);
 406 
 407   __ bind(resolved);
 408 
 409   { // Check for the null sentinel.
 410     // If we just called the VM, it already did the mapping for us,
 411     // but it's harmless to retry.
 412     Label notNull;
 413 
 414     // Stash null_sentinel address to get its value later
 415     __ movptr(rarg, (uintptr_t)Universe::the_null_sentinel_addr());
 416     __ ldr(tmp, Address(rarg));
 417     __ cmp(result, tmp);
 418     __ br(Assembler::NE, notNull);
 419     __ mov(result, 0);  // NULL object reference
 420     __ bind(notNull);
 421   }
 422 
 423   if (VerifyOops) {
 424     // Safe to call with 0 result
 425     __ verify_oop(result);
 426   }
 427 }
 428 
 429 void TemplateTable::ldc2_w()
 430 {
 431   transition(vtos, vtos);
 432   Label notDouble, notLong, Done;
 433   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 434 
 435   __ get_cpool_and_tags(r1, r2);
 436   const int base_offset = ConstantPool::header_size() * wordSize;
 437   const int tags_offset = Array<u1>::base_offset_in_bytes();
 438 
 439   // get type
 440   __ lea(r2, Address(r2, r0, Address::lsl(0)));
 441   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 442   __ cmpw(r2, (int)JVM_CONSTANT_Double);
 443   __ br(Assembler::NE, notDouble);
 444 
 445   // dtos
 446   __ lea (r2, Address(r1, r0, Address::lsl(3)));
 447   __ ldrd(v0, Address(r2, base_offset));
 448   __ push_d();
 449   __ b(Done);
 450 
 451   __ bind(notDouble);
 452   __ cmpw(r2, (int)JVM_CONSTANT_Long);
 453   __ br(Assembler::NE, notLong);
 454 
 455   // ltos
 456   __ lea(r0, Address(r1, r0, Address::lsl(3)));
 457   __ ldr(r0, Address(r0, base_offset));
 458   __ push_l();
 459   __ b(Done);
 460 
 461   __ bind(notLong);
 462   condy_helper(Done);
 463 
 464   __ bind(Done);
 465 }
 466 
 467 void TemplateTable::condy_helper(Label& Done)
 468 {
 469   Register obj = r0;
 470   Register rarg = r1;
 471   Register flags = r2;
 472   Register off = r3;
 473 
 474   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 475 
 476   __ mov(rarg, (int) bytecode());
 477   __ call_VM(obj, entry, rarg);
 478 
 479   __ get_vm_result_2(flags, rthread);
 480 
 481   // VMr = obj = base address to find primitive value to push
 482   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 483   __ mov(off, flags);
 484   __ andw(off, off, ConstantPoolCacheEntry::field_index_mask);
 485 
 486   const Address field(obj, off);
 487 
 488   // What sort of thing are we loading?
 489   // x86 uses a shift and mask or wings it with a shift plus assert
 490   // the mask is not needed. aarch64 just uses bitfield extract
 491   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,
 492            ConstantPoolCacheEntry::tos_state_bits);
 493 
 494   switch (bytecode()) {
 495     case Bytecodes::_ldc:
 496     case Bytecodes::_ldc_w:
 497       {
 498         // tos in (itos, ftos, stos, btos, ctos, ztos)
 499         Label notInt, notFloat, notShort, notByte, notChar, notBool;
 500         __ cmpw(flags, itos);
 501         __ br(Assembler::NE, notInt);
 502         // itos
 503         __ ldrw(r0, field);
 504         __ push(itos);
 505         __ b(Done);
 506 
 507         __ bind(notInt);
 508         __ cmpw(flags, ftos);
 509         __ br(Assembler::NE, notFloat);
 510         // ftos
 511         __ load_float(field);
 512         __ push(ftos);
 513         __ b(Done);
 514 
 515         __ bind(notFloat);
 516         __ cmpw(flags, stos);
 517         __ br(Assembler::NE, notShort);
 518         // stos
 519         __ load_signed_short(r0, field);
 520         __ push(stos);
 521         __ b(Done);
 522 
 523         __ bind(notShort);
 524         __ cmpw(flags, btos);
 525         __ br(Assembler::NE, notByte);
 526         // btos
 527         __ load_signed_byte(r0, field);
 528         __ push(btos);
 529         __ b(Done);
 530 
 531         __ bind(notByte);
 532         __ cmpw(flags, ctos);
 533         __ br(Assembler::NE, notChar);
 534         // ctos
 535         __ load_unsigned_short(r0, field);
 536         __ push(ctos);
 537         __ b(Done);
 538 
 539         __ bind(notChar);
 540         __ cmpw(flags, ztos);
 541         __ br(Assembler::NE, notBool);
 542         // ztos
 543         __ load_signed_byte(r0, field);
 544         __ push(ztos);
 545         __ b(Done);
 546 
 547         __ bind(notBool);
 548         break;
 549       }
 550 
 551     case Bytecodes::_ldc2_w:
 552       {
 553         Label notLong, notDouble;
 554         __ cmpw(flags, ltos);
 555         __ br(Assembler::NE, notLong);
 556         // ltos
 557         __ ldr(r0, field);
 558         __ push(ltos);
 559         __ b(Done);
 560 
 561         __ bind(notLong);
 562         __ cmpw(flags, dtos);
 563         __ br(Assembler::NE, notDouble);
 564         // dtos
 565         __ load_double(field);
 566         __ push(dtos);
 567         __ b(Done);
 568 
 569        __ bind(notDouble);
 570         break;
 571       }
 572 
 573     default:
 574       ShouldNotReachHere();
 575     }
 576 
 577     __ stop("bad ldc/condy");
 578 }
 579 
 580 void TemplateTable::locals_index(Register reg, int offset)
 581 {
 582   __ ldrb(reg, at_bcp(offset));
 583   __ neg(reg, reg);
 584 }
 585 
 586 void TemplateTable::iload() {
 587   iload_internal();
 588 }
 589 
 590 void TemplateTable::nofast_iload() {
 591   iload_internal(may_not_rewrite);
 592 }
 593 
 594 void TemplateTable::iload_internal(RewriteControl rc) {
 595   transition(vtos, itos);
 596   if (RewriteFrequentPairs && rc == may_rewrite) {
 597     Label rewrite, done;
 598     Register bc = r4;
 599 
 600     // get next bytecode
 601     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 602 
 603     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 604     // last two iloads in a pair.  Comparing against fast_iload means that
 605     // the next bytecode is neither an iload or a caload, and therefore
 606     // an iload pair.
 607     __ cmpw(r1, Bytecodes::_iload);
 608     __ br(Assembler::EQ, done);
 609 
 610     // if _fast_iload rewrite to _fast_iload2
 611     __ cmpw(r1, Bytecodes::_fast_iload);
 612     __ movw(bc, Bytecodes::_fast_iload2);
 613     __ br(Assembler::EQ, rewrite);
 614 
 615     // if _caload rewrite to _fast_icaload
 616     __ cmpw(r1, Bytecodes::_caload);
 617     __ movw(bc, Bytecodes::_fast_icaload);
 618     __ br(Assembler::EQ, rewrite);
 619 
 620     // else rewrite to _fast_iload
 621     __ movw(bc, Bytecodes::_fast_iload);
 622 
 623     // rewrite
 624     // bc: new bytecode
 625     __ bind(rewrite);
 626     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 627     __ bind(done);
 628 
 629   }
 630 
 631   // do iload, get the local value into tos
 632   locals_index(r1);
 633   __ ldr(r0, iaddress(r1));
 634 
 635 }
 636 
 637 void TemplateTable::fast_iload2()
 638 {
 639   transition(vtos, itos);
 640   locals_index(r1);
 641   __ ldr(r0, iaddress(r1));
 642   __ push(itos);
 643   locals_index(r1, 3);
 644   __ ldr(r0, iaddress(r1));
 645 }
 646 
 647 void TemplateTable::fast_iload()
 648 {
 649   transition(vtos, itos);
 650   locals_index(r1);
 651   __ ldr(r0, iaddress(r1));
 652 }
 653 
 654 void TemplateTable::lload()
 655 {
 656   transition(vtos, ltos);
 657   __ ldrb(r1, at_bcp(1));
 658   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 659   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 660 }
 661 
 662 void TemplateTable::fload()
 663 {
 664   transition(vtos, ftos);
 665   locals_index(r1);
 666   // n.b. we use ldrd here because this is a 64 bit slot
 667   // this is comparable to the iload case
 668   __ ldrd(v0, faddress(r1));
 669 }
 670 
 671 void TemplateTable::dload()
 672 {
 673   transition(vtos, dtos);
 674   __ ldrb(r1, at_bcp(1));
 675   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 676   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 677 }
 678 
 679 void TemplateTable::aload()
 680 {
 681   transition(vtos, atos);
 682   locals_index(r1);
 683   __ ldr(r0, iaddress(r1));
 684 }
 685 
 686 void TemplateTable::locals_index_wide(Register reg) {
 687   __ ldrh(reg, at_bcp(2));
 688   __ rev16w(reg, reg);
 689   __ neg(reg, reg);
 690 }
 691 
 692 void TemplateTable::wide_iload() {
 693   transition(vtos, itos);
 694   locals_index_wide(r1);
 695   __ ldr(r0, iaddress(r1));
 696 }
 697 
 698 void TemplateTable::wide_lload()
 699 {
 700   transition(vtos, ltos);
 701   __ ldrh(r1, at_bcp(2));
 702   __ rev16w(r1, r1);
 703   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 704   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 705 }
 706 
 707 void TemplateTable::wide_fload()
 708 {
 709   transition(vtos, ftos);
 710   locals_index_wide(r1);
 711   // n.b. we use ldrd here because this is a 64 bit slot
 712   // this is comparable to the iload case
 713   __ ldrd(v0, faddress(r1));
 714 }
 715 
 716 void TemplateTable::wide_dload()
 717 {
 718   transition(vtos, dtos);
 719   __ ldrh(r1, at_bcp(2));
 720   __ rev16w(r1, r1);
 721   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 722   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 723 }
 724 
 725 void TemplateTable::wide_aload()
 726 {
 727   transition(vtos, atos);
 728   locals_index_wide(r1);
 729   __ ldr(r0, aaddress(r1));
 730 }
 731 
 732 void TemplateTable::index_check(Register array, Register index)
 733 {
 734   // destroys r1, rscratch1
 735   // check array
 736   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 737   // sign extend index for use by indexed load
 738   // __ movl2ptr(index, index);
 739   // check index
 740   Register length = rscratch1;
 741   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 742   __ cmpw(index, length);
 743   if (index != r1) {
 744     // ??? convention: move aberrant index into r1 for exception message
 745     assert(r1 != array, "different registers");
 746     __ mov(r1, index);
 747   }
 748   Label ok;
 749   __ br(Assembler::LO, ok);
 750   __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 751   __ br(rscratch1);
 752   __ bind(ok);
 753 }
 754 
 755 void TemplateTable::iaload()
 756 {
 757   transition(itos, itos);
 758   __ mov(r1, r0);
 759   __ pop_ptr(r0);
 760   // r0: array
 761   // r1: index
 762   index_check(r0, r1); // leaves index in r1, kills rscratch1
 763   __ lea(r1, Address(r0, r1, Address::uxtw(2)));
 764   __ ldrw(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_INT)));
 765 }
 766 
 767 void TemplateTable::laload()
 768 {
 769   transition(itos, ltos);
 770   __ mov(r1, r0);
 771   __ pop_ptr(r0);
 772   // r0: array
 773   // r1: index
 774   index_check(r0, r1); // leaves index in r1, kills rscratch1
 775   __ lea(r1, Address(r0, r1, Address::uxtw(3)));
 776   __ ldr(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_LONG)));
 777 }
 778 
 779 void TemplateTable::faload()
 780 {
 781   transition(itos, ftos);
 782   __ mov(r1, r0);
 783   __ pop_ptr(r0);
 784   // r0: array
 785   // r1: index
 786   index_check(r0, r1); // leaves index in r1, kills rscratch1
 787   __ lea(r1,  Address(r0, r1, Address::uxtw(2)));
 788   __ ldrs(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
 789 }
 790 
 791 void TemplateTable::daload()
 792 {
 793   transition(itos, dtos);
 794   __ mov(r1, r0);
 795   __ pop_ptr(r0);
 796   // r0: array
 797   // r1: index
 798   index_check(r0, r1); // leaves index in r1, kills rscratch1
 799   __ lea(r1,  Address(r0, r1, Address::uxtw(3)));
 800   __ ldrd(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
 801 }
 802 
 803 void TemplateTable::aaload()
 804 {
 805   transition(itos, atos);
 806   __ mov(r1, r0);
 807   __ pop_ptr(r0);
 808   // r0: array
 809   // r1: index
 810   index_check(r0, r1); // leaves index in r1, kills rscratch1
 811   int s = (UseCompressedOops ? 2 : 3);
 812   __ lea(r1, Address(r0, r1, Address::uxtw(s)));
 813   do_oop_load(_masm,
 814               Address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
 815               r0,
 816               IN_HEAP | IN_HEAP_ARRAY);
 817 }
 818 
 819 void TemplateTable::baload()
 820 {
 821   transition(itos, itos);
 822   __ mov(r1, r0);
 823   __ pop_ptr(r0);
 824   // r0: array
 825   // r1: index
 826   index_check(r0, r1); // leaves index in r1, kills rscratch1
 827   __ lea(r1,  Address(r0, r1, Address::uxtw(0)));
 828   __ load_signed_byte(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 829 }
 830 
 831 void TemplateTable::caload()
 832 {
 833   transition(itos, itos);
 834   __ mov(r1, r0);
 835   __ pop_ptr(r0);
 836   // r0: array
 837   // r1: index
 838   index_check(r0, r1); // leaves index in r1, kills rscratch1
 839   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 840   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 841 }
 842 
 843 // iload followed by caload frequent pair
 844 void TemplateTable::fast_icaload()
 845 {
 846   transition(vtos, itos);
 847   // load index out of locals
 848   locals_index(r2);
 849   __ ldr(r1, iaddress(r2));
 850 
 851   __ pop_ptr(r0);
 852 
 853   // r0: array
 854   // r1: index
 855   index_check(r0, r1); // leaves index in r1, kills rscratch1
 856   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 857   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 858 }
 859 
 860 void TemplateTable::saload()
 861 {
 862   transition(itos, itos);
 863   __ mov(r1, r0);
 864   __ pop_ptr(r0);
 865   // r0: array
 866   // r1: index
 867   index_check(r0, r1); // leaves index in r1, kills rscratch1
 868   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 869   __ load_signed_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 870 }
 871 
 872 void TemplateTable::iload(int n)
 873 {
 874   transition(vtos, itos);
 875   __ ldr(r0, iaddress(n));
 876 }
 877 
 878 void TemplateTable::lload(int n)
 879 {
 880   transition(vtos, ltos);
 881   __ ldr(r0, laddress(n));
 882 }
 883 
 884 void TemplateTable::fload(int n)
 885 {
 886   transition(vtos, ftos);
 887   __ ldrs(v0, faddress(n));
 888 }
 889 
 890 void TemplateTable::dload(int n)
 891 {
 892   transition(vtos, dtos);
 893   __ ldrd(v0, daddress(n));
 894 }
 895 
 896 void TemplateTable::aload(int n)
 897 {
 898   transition(vtos, atos);
 899   __ ldr(r0, iaddress(n));
 900 }
 901 
 902 void TemplateTable::aload_0() {
 903   aload_0_internal();
 904 }
 905 
 906 void TemplateTable::nofast_aload_0() {
 907   aload_0_internal(may_not_rewrite);
 908 }
 909 
 910 void TemplateTable::aload_0_internal(RewriteControl rc) {
 911   // According to bytecode histograms, the pairs:
 912   //
 913   // _aload_0, _fast_igetfield
 914   // _aload_0, _fast_agetfield
 915   // _aload_0, _fast_fgetfield
 916   //
 917   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 918   // _aload_0 bytecode checks if the next bytecode is either
 919   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 920   // rewrites the current bytecode into a pair bytecode; otherwise it
 921   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 922   // the pair check anymore.
 923   //
 924   // Note: If the next bytecode is _getfield, the rewrite must be
 925   //       delayed, otherwise we may miss an opportunity for a pair.
 926   //
 927   // Also rewrite frequent pairs
 928   //   aload_0, aload_1
 929   //   aload_0, iload_1
 930   // These bytecodes with a small amount of code are most profitable
 931   // to rewrite
 932   if (RewriteFrequentPairs && rc == may_rewrite) {
 933     Label rewrite, done;
 934     const Register bc = r4;
 935 
 936     // get next bytecode
 937     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 938 
 939     // if _getfield then wait with rewrite
 940     __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
 941     __ br(Assembler::EQ, done);
 942 
 943     // if _igetfield then rewrite to _fast_iaccess_0
 944     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 945     __ cmpw(r1, Bytecodes::_fast_igetfield);
 946     __ movw(bc, Bytecodes::_fast_iaccess_0);
 947     __ br(Assembler::EQ, rewrite);
 948 
 949     // if _agetfield then rewrite to _fast_aaccess_0
 950     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 951     __ cmpw(r1, Bytecodes::_fast_agetfield);
 952     __ movw(bc, Bytecodes::_fast_aaccess_0);
 953     __ br(Assembler::EQ, rewrite);
 954 
 955     // if _fgetfield then rewrite to _fast_faccess_0
 956     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 957     __ cmpw(r1, Bytecodes::_fast_fgetfield);
 958     __ movw(bc, Bytecodes::_fast_faccess_0);
 959     __ br(Assembler::EQ, rewrite);
 960 
 961     // else rewrite to _fast_aload0
 962     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 963     __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
 964 
 965     // rewrite
 966     // bc: new bytecode
 967     __ bind(rewrite);
 968     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 969 
 970     __ bind(done);
 971   }
 972 
 973   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 974   aload(0);
 975 }
 976 
 977 void TemplateTable::istore()
 978 {
 979   transition(itos, vtos);
 980   locals_index(r1);
 981   // FIXME: We're being very pernickerty here storing a jint in a
 982   // local with strw, which costs an extra instruction over what we'd
 983   // be able to do with a simple str.  We should just store the whole
 984   // word.
 985   __ lea(rscratch1, iaddress(r1));
 986   __ strw(r0, Address(rscratch1));
 987 }
 988 
 989 void TemplateTable::lstore()
 990 {
 991   transition(ltos, vtos);
 992   locals_index(r1);
 993   __ str(r0, laddress(r1, rscratch1, _masm));
 994 }
 995 
 996 void TemplateTable::fstore() {
 997   transition(ftos, vtos);
 998   locals_index(r1);
 999   __ lea(rscratch1, iaddress(r1));
1000   __ strs(v0, Address(rscratch1));
1001 }
1002 
1003 void TemplateTable::dstore() {
1004   transition(dtos, vtos);
1005   locals_index(r1);
1006   __ strd(v0, daddress(r1, rscratch1, _masm));
1007 }
1008 
1009 void TemplateTable::astore()
1010 {
1011   transition(vtos, vtos);
1012   __ pop_ptr(r0);
1013   locals_index(r1);
1014   __ str(r0, aaddress(r1));
1015 }
1016 
1017 void TemplateTable::wide_istore() {
1018   transition(vtos, vtos);
1019   __ pop_i();
1020   locals_index_wide(r1);
1021   __ lea(rscratch1, iaddress(r1));
1022   __ strw(r0, Address(rscratch1));
1023 }
1024 
1025 void TemplateTable::wide_lstore() {
1026   transition(vtos, vtos);
1027   __ pop_l();
1028   locals_index_wide(r1);
1029   __ str(r0, laddress(r1, rscratch1, _masm));
1030 }
1031 
1032 void TemplateTable::wide_fstore() {
1033   transition(vtos, vtos);
1034   __ pop_f();
1035   locals_index_wide(r1);
1036   __ lea(rscratch1, faddress(r1));
1037   __ strs(v0, rscratch1);
1038 }
1039 
1040 void TemplateTable::wide_dstore() {
1041   transition(vtos, vtos);
1042   __ pop_d();
1043   locals_index_wide(r1);
1044   __ strd(v0, daddress(r1, rscratch1, _masm));
1045 }
1046 
1047 void TemplateTable::wide_astore() {
1048   transition(vtos, vtos);
1049   __ pop_ptr(r0);
1050   locals_index_wide(r1);
1051   __ str(r0, aaddress(r1));
1052 }
1053 
1054 void TemplateTable::iastore() {
1055   transition(itos, vtos);
1056   __ pop_i(r1);
1057   __ pop_ptr(r3);
1058   // r0: value
1059   // r1: index
1060   // r3: array
1061   index_check(r3, r1); // prefer index in r1
1062   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1063   __ strw(r0, Address(rscratch1,
1064                       arrayOopDesc::base_offset_in_bytes(T_INT)));
1065 }
1066 
1067 void TemplateTable::lastore() {
1068   transition(ltos, vtos);
1069   __ pop_i(r1);
1070   __ pop_ptr(r3);
1071   // r0: value
1072   // r1: index
1073   // r3: array
1074   index_check(r3, r1); // prefer index in r1
1075   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1076   __ str(r0, Address(rscratch1,
1077                       arrayOopDesc::base_offset_in_bytes(T_LONG)));
1078 }
1079 
1080 void TemplateTable::fastore() {
1081   transition(ftos, vtos);
1082   __ pop_i(r1);
1083   __ pop_ptr(r3);
1084   // v0: value
1085   // r1:  index
1086   // r3:  array
1087   index_check(r3, r1); // prefer index in r1
1088   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1089   __ strs(v0, Address(rscratch1,
1090                       arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
1091 }
1092 
1093 void TemplateTable::dastore() {
1094   transition(dtos, vtos);
1095   __ pop_i(r1);
1096   __ pop_ptr(r3);
1097   // v0: value
1098   // r1:  index
1099   // r3:  array
1100   index_check(r3, r1); // prefer index in r1
1101   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1102   __ strd(v0, Address(rscratch1,
1103                       arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1104 }
1105 
1106 void TemplateTable::aastore() {
1107   Label is_null, ok_is_subtype, done;
1108   transition(vtos, vtos);
1109   // stack: ..., array, index, value
1110   __ ldr(r0, at_tos());    // value
1111   __ ldr(r2, at_tos_p1()); // index
1112   __ ldr(r3, at_tos_p2()); // array
1113 
1114   Address element_address(r4, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1115 
1116   index_check(r3, r2);     // kills r1
1117   __ lea(r4, Address(r3, r2, Address::uxtw(UseCompressedOops? 2 : 3)));
1118 
1119   // do array store check - check for NULL value first
1120   __ cbz(r0, is_null);
1121 
1122   // Move subklass into r1
1123   __ load_klass(r1, r0);
1124   // Move superklass into r0
1125   __ load_klass(r0, r3);
1126   __ ldr(r0, Address(r0,
1127                      ObjArrayKlass::element_klass_offset()));
1128   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1129 
1130   // Generate subtype check.  Blows r2, r5
1131   // Superklass in r0.  Subklass in r1.
1132   __ gen_subtype_check(r1, ok_is_subtype);
1133 
1134   // Come here on failure
1135   // object is at TOS
1136   __ b(Interpreter::_throw_ArrayStoreException_entry);
1137 
1138   // Come here on success
1139   __ bind(ok_is_subtype);
1140 
1141   // Get the value we will store
1142   __ ldr(r0, at_tos());
1143   // Now store using the appropriate barrier
1144   do_oop_store(_masm, element_address, r0, IN_HEAP | IN_HEAP_ARRAY);
1145   __ b(done);
1146 
1147   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1148   __ bind(is_null);
1149   __ profile_null_seen(r2);
1150 
1151   // Store a NULL
1152   do_oop_store(_masm, element_address, noreg, IN_HEAP | IN_HEAP_ARRAY);
1153 
1154   // Pop stack arguments
1155   __ bind(done);
1156   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1157 }
1158 
1159 void TemplateTable::bastore()
1160 {
1161   transition(itos, vtos);
1162   __ pop_i(r1);
1163   __ pop_ptr(r3);
1164   // r0: value
1165   // r1: index
1166   // r3: array
1167   index_check(r3, r1); // prefer index in r1
1168 
1169   // Need to check whether array is boolean or byte
1170   // since both types share the bastore bytecode.
1171   __ load_klass(r2, r3);
1172   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1173   int diffbit_index = exact_log2(Klass::layout_helper_boolean_diffbit());
1174   Label L_skip;
1175   __ tbz(r2, diffbit_index, L_skip);
1176   __ andw(r0, r0, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1177   __ bind(L_skip);
1178 
1179   __ lea(rscratch1, Address(r3, r1, Address::uxtw(0)));
1180   __ strb(r0, Address(rscratch1,
1181                       arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1182 }
1183 
1184 void TemplateTable::castore()
1185 {
1186   transition(itos, vtos);
1187   __ pop_i(r1);
1188   __ pop_ptr(r3);
1189   // r0: value
1190   // r1: index
1191   // r3: array
1192   index_check(r3, r1); // prefer index in r1
1193   __ lea(rscratch1, Address(r3, r1, Address::uxtw(1)));
1194   __ strh(r0, Address(rscratch1,
1195                       arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1196 }
1197 
1198 void TemplateTable::sastore()
1199 {
1200   castore();
1201 }
1202 
1203 void TemplateTable::istore(int n)
1204 {
1205   transition(itos, vtos);
1206   __ str(r0, iaddress(n));
1207 }
1208 
1209 void TemplateTable::lstore(int n)
1210 {
1211   transition(ltos, vtos);
1212   __ str(r0, laddress(n));
1213 }
1214 
1215 void TemplateTable::fstore(int n)
1216 {
1217   transition(ftos, vtos);
1218   __ strs(v0, faddress(n));
1219 }
1220 
1221 void TemplateTable::dstore(int n)
1222 {
1223   transition(dtos, vtos);
1224   __ strd(v0, daddress(n));
1225 }
1226 
1227 void TemplateTable::astore(int n)
1228 {
1229   transition(vtos, vtos);
1230   __ pop_ptr(r0);
1231   __ str(r0, iaddress(n));
1232 }
1233 
1234 void TemplateTable::pop()
1235 {
1236   transition(vtos, vtos);
1237   __ add(esp, esp, Interpreter::stackElementSize);
1238 }
1239 
1240 void TemplateTable::pop2()
1241 {
1242   transition(vtos, vtos);
1243   __ add(esp, esp, 2 * Interpreter::stackElementSize);
1244 }
1245 
1246 void TemplateTable::dup()
1247 {
1248   transition(vtos, vtos);
1249   __ ldr(r0, Address(esp, 0));
1250   __ push(r0);
1251   // stack: ..., a, a
1252 }
1253 
1254 void TemplateTable::dup_x1()
1255 {
1256   transition(vtos, vtos);
1257   // stack: ..., a, b
1258   __ ldr(r0, at_tos());  // load b
1259   __ ldr(r2, at_tos_p1());  // load a
1260   __ str(r0, at_tos_p1());  // store b
1261   __ str(r2, at_tos());  // store a
1262   __ push(r0);                  // push b
1263   // stack: ..., b, a, b
1264 }
1265 
1266 void TemplateTable::dup_x2()
1267 {
1268   transition(vtos, vtos);
1269   // stack: ..., a, b, c
1270   __ ldr(r0, at_tos());  // load c
1271   __ ldr(r2, at_tos_p2());  // load a
1272   __ str(r0, at_tos_p2());  // store c in a
1273   __ push(r0);      // push c
1274   // stack: ..., c, b, c, c
1275   __ ldr(r0, at_tos_p2());  // load b
1276   __ str(r2, at_tos_p2());  // store a in b
1277   // stack: ..., c, a, c, c
1278   __ str(r0, at_tos_p1());  // store b in c
1279   // stack: ..., c, a, b, c
1280 }
1281 
1282 void TemplateTable::dup2()
1283 {
1284   transition(vtos, vtos);
1285   // stack: ..., a, b
1286   __ ldr(r0, at_tos_p1());  // load a
1287   __ push(r0);                  // push a
1288   __ ldr(r0, at_tos_p1());  // load b
1289   __ push(r0);                  // push b
1290   // stack: ..., a, b, a, b
1291 }
1292 
1293 void TemplateTable::dup2_x1()
1294 {
1295   transition(vtos, vtos);
1296   // stack: ..., a, b, c
1297   __ ldr(r2, at_tos());  // load c
1298   __ ldr(r0, at_tos_p1());  // load b
1299   __ push(r0);                  // push b
1300   __ push(r2);                  // push c
1301   // stack: ..., a, b, c, b, c
1302   __ str(r2, at_tos_p3());  // store c in b
1303   // stack: ..., a, c, c, b, c
1304   __ ldr(r2, at_tos_p4());  // load a
1305   __ str(r2, at_tos_p2());  // store a in 2nd c
1306   // stack: ..., a, c, a, b, c
1307   __ str(r0, at_tos_p4());  // store b in a
1308   // stack: ..., b, c, a, b, c
1309 }
1310 
1311 void TemplateTable::dup2_x2()
1312 {
1313   transition(vtos, vtos);
1314   // stack: ..., a, b, c, d
1315   __ ldr(r2, at_tos());  // load d
1316   __ ldr(r0, at_tos_p1());  // load c
1317   __ push(r0)            ;      // push c
1318   __ push(r2);                  // push d
1319   // stack: ..., a, b, c, d, c, d
1320   __ ldr(r0, at_tos_p4());  // load b
1321   __ str(r0, at_tos_p2());  // store b in d
1322   __ str(r2, at_tos_p4());  // store d in b
1323   // stack: ..., a, d, c, b, c, d
1324   __ ldr(r2, at_tos_p5());  // load a
1325   __ ldr(r0, at_tos_p3());  // load c
1326   __ str(r2, at_tos_p3());  // store a in c
1327   __ str(r0, at_tos_p5());  // store c in a
1328   // stack: ..., c, d, a, b, c, d
1329 }
1330 
1331 void TemplateTable::swap()
1332 {
1333   transition(vtos, vtos);
1334   // stack: ..., a, b
1335   __ ldr(r2, at_tos_p1());  // load a
1336   __ ldr(r0, at_tos());  // load b
1337   __ str(r2, at_tos());  // store a in b
1338   __ str(r0, at_tos_p1());  // store b in a
1339   // stack: ..., b, a
1340 }
1341 
1342 void TemplateTable::iop2(Operation op)
1343 {
1344   transition(itos, itos);
1345   // r0 <== r1 op r0
1346   __ pop_i(r1);
1347   switch (op) {
1348   case add  : __ addw(r0, r1, r0); break;
1349   case sub  : __ subw(r0, r1, r0); break;
1350   case mul  : __ mulw(r0, r1, r0); break;
1351   case _and : __ andw(r0, r1, r0); break;
1352   case _or  : __ orrw(r0, r1, r0); break;
1353   case _xor : __ eorw(r0, r1, r0); break;
1354   case shl  : __ lslvw(r0, r1, r0); break;
1355   case shr  : __ asrvw(r0, r1, r0); break;
1356   case ushr : __ lsrvw(r0, r1, r0);break;
1357   default   : ShouldNotReachHere();
1358   }
1359 }
1360 
1361 void TemplateTable::lop2(Operation op)
1362 {
1363   transition(ltos, ltos);
1364   // r0 <== r1 op r0
1365   __ pop_l(r1);
1366   switch (op) {
1367   case add  : __ add(r0, r1, r0); break;
1368   case sub  : __ sub(r0, r1, r0); break;
1369   case mul  : __ mul(r0, r1, r0); break;
1370   case _and : __ andr(r0, r1, r0); break;
1371   case _or  : __ orr(r0, r1, r0); break;
1372   case _xor : __ eor(r0, r1, r0); break;
1373   default   : ShouldNotReachHere();
1374   }
1375 }
1376 
1377 void TemplateTable::idiv()
1378 {
1379   transition(itos, itos);
1380   // explicitly check for div0
1381   Label no_div0;
1382   __ cbnzw(r0, no_div0);
1383   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1384   __ br(rscratch1);
1385   __ bind(no_div0);
1386   __ pop_i(r1);
1387   // r0 <== r1 idiv r0
1388   __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1389 }
1390 
1391 void TemplateTable::irem()
1392 {
1393   transition(itos, itos);
1394   // explicitly check for div0
1395   Label no_div0;
1396   __ cbnzw(r0, no_div0);
1397   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1398   __ br(rscratch1);
1399   __ bind(no_div0);
1400   __ pop_i(r1);
1401   // r0 <== r1 irem r0
1402   __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1403 }
1404 
1405 void TemplateTable::lmul()
1406 {
1407   transition(ltos, ltos);
1408   __ pop_l(r1);
1409   __ mul(r0, r0, r1);
1410 }
1411 
1412 void TemplateTable::ldiv()
1413 {
1414   transition(ltos, ltos);
1415   // explicitly check for div0
1416   Label no_div0;
1417   __ cbnz(r0, no_div0);
1418   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1419   __ br(rscratch1);
1420   __ bind(no_div0);
1421   __ pop_l(r1);
1422   // r0 <== r1 ldiv r0
1423   __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1424 }
1425 
1426 void TemplateTable::lrem()
1427 {
1428   transition(ltos, ltos);
1429   // explicitly check for div0
1430   Label no_div0;
1431   __ cbnz(r0, no_div0);
1432   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1433   __ br(rscratch1);
1434   __ bind(no_div0);
1435   __ pop_l(r1);
1436   // r0 <== r1 lrem r0
1437   __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1438 }
1439 
1440 void TemplateTable::lshl()
1441 {
1442   transition(itos, ltos);
1443   // shift count is in r0
1444   __ pop_l(r1);
1445   __ lslv(r0, r1, r0);
1446 }
1447 
1448 void TemplateTable::lshr()
1449 {
1450   transition(itos, ltos);
1451   // shift count is in r0
1452   __ pop_l(r1);
1453   __ asrv(r0, r1, r0);
1454 }
1455 
1456 void TemplateTable::lushr()
1457 {
1458   transition(itos, ltos);
1459   // shift count is in r0
1460   __ pop_l(r1);
1461   __ lsrv(r0, r1, r0);
1462 }
1463 
1464 void TemplateTable::fop2(Operation op)
1465 {
1466   transition(ftos, ftos);
1467   switch (op) {
1468   case add:
1469     // n.b. use ldrd because this is a 64 bit slot
1470     __ pop_f(v1);
1471     __ fadds(v0, v1, v0);
1472     break;
1473   case sub:
1474     __ pop_f(v1);
1475     __ fsubs(v0, v1, v0);
1476     break;
1477   case mul:
1478     __ pop_f(v1);
1479     __ fmuls(v0, v1, v0);
1480     break;
1481   case div:
1482     __ pop_f(v1);
1483     __ fdivs(v0, v1, v0);
1484     break;
1485   case rem:
1486     __ fmovs(v1, v0);
1487     __ pop_f(v0);
1488     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::frem),
1489                          0, 2, MacroAssembler::ret_type_float);
1490     break;
1491   default:
1492     ShouldNotReachHere();
1493     break;
1494   }
1495 }
1496 
1497 void TemplateTable::dop2(Operation op)
1498 {
1499   transition(dtos, dtos);
1500   switch (op) {
1501   case add:
1502     // n.b. use ldrd because this is a 64 bit slot
1503     __ pop_d(v1);
1504     __ faddd(v0, v1, v0);
1505     break;
1506   case sub:
1507     __ pop_d(v1);
1508     __ fsubd(v0, v1, v0);
1509     break;
1510   case mul:
1511     __ pop_d(v1);
1512     __ fmuld(v0, v1, v0);
1513     break;
1514   case div:
1515     __ pop_d(v1);
1516     __ fdivd(v0, v1, v0);
1517     break;
1518   case rem:
1519     __ fmovd(v1, v0);
1520     __ pop_d(v0);
1521     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::drem),
1522                          0, 2, MacroAssembler::ret_type_double);
1523     break;
1524   default:
1525     ShouldNotReachHere();
1526     break;
1527   }
1528 }
1529 
1530 void TemplateTable::ineg()
1531 {
1532   transition(itos, itos);
1533   __ negw(r0, r0);
1534 
1535 }
1536 
1537 void TemplateTable::lneg()
1538 {
1539   transition(ltos, ltos);
1540   __ neg(r0, r0);
1541 }
1542 
1543 void TemplateTable::fneg()
1544 {
1545   transition(ftos, ftos);
1546   __ fnegs(v0, v0);
1547 }
1548 
1549 void TemplateTable::dneg()
1550 {
1551   transition(dtos, dtos);
1552   __ fnegd(v0, v0);
1553 }
1554 
1555 void TemplateTable::iinc()
1556 {
1557   transition(vtos, vtos);
1558   __ load_signed_byte(r1, at_bcp(2)); // get constant
1559   locals_index(r2);
1560   __ ldr(r0, iaddress(r2));
1561   __ addw(r0, r0, r1);
1562   __ str(r0, iaddress(r2));
1563 }
1564 
1565 void TemplateTable::wide_iinc()
1566 {
1567   transition(vtos, vtos);
1568   // __ mov(r1, zr);
1569   __ ldrw(r1, at_bcp(2)); // get constant and index
1570   __ rev16(r1, r1);
1571   __ ubfx(r2, r1, 0, 16);
1572   __ neg(r2, r2);
1573   __ sbfx(r1, r1, 16, 16);
1574   __ ldr(r0, iaddress(r2));
1575   __ addw(r0, r0, r1);
1576   __ str(r0, iaddress(r2));
1577 }
1578 
1579 void TemplateTable::convert()
1580 {
1581   // Checking
1582 #ifdef ASSERT
1583   {
1584     TosState tos_in  = ilgl;
1585     TosState tos_out = ilgl;
1586     switch (bytecode()) {
1587     case Bytecodes::_i2l: // fall through
1588     case Bytecodes::_i2f: // fall through
1589     case Bytecodes::_i2d: // fall through
1590     case Bytecodes::_i2b: // fall through
1591     case Bytecodes::_i2c: // fall through
1592     case Bytecodes::_i2s: tos_in = itos; break;
1593     case Bytecodes::_l2i: // fall through
1594     case Bytecodes::_l2f: // fall through
1595     case Bytecodes::_l2d: tos_in = ltos; break;
1596     case Bytecodes::_f2i: // fall through
1597     case Bytecodes::_f2l: // fall through
1598     case Bytecodes::_f2d: tos_in = ftos; break;
1599     case Bytecodes::_d2i: // fall through
1600     case Bytecodes::_d2l: // fall through
1601     case Bytecodes::_d2f: tos_in = dtos; break;
1602     default             : ShouldNotReachHere();
1603     }
1604     switch (bytecode()) {
1605     case Bytecodes::_l2i: // fall through
1606     case Bytecodes::_f2i: // fall through
1607     case Bytecodes::_d2i: // fall through
1608     case Bytecodes::_i2b: // fall through
1609     case Bytecodes::_i2c: // fall through
1610     case Bytecodes::_i2s: tos_out = itos; break;
1611     case Bytecodes::_i2l: // fall through
1612     case Bytecodes::_f2l: // fall through
1613     case Bytecodes::_d2l: tos_out = ltos; break;
1614     case Bytecodes::_i2f: // fall through
1615     case Bytecodes::_l2f: // fall through
1616     case Bytecodes::_d2f: tos_out = ftos; break;
1617     case Bytecodes::_i2d: // fall through
1618     case Bytecodes::_l2d: // fall through
1619     case Bytecodes::_f2d: tos_out = dtos; break;
1620     default             : ShouldNotReachHere();
1621     }
1622     transition(tos_in, tos_out);
1623   }
1624 #endif // ASSERT
1625   // static const int64_t is_nan = 0x8000000000000000L;
1626 
1627   // Conversion
1628   switch (bytecode()) {
1629   case Bytecodes::_i2l:
1630     __ sxtw(r0, r0);
1631     break;
1632   case Bytecodes::_i2f:
1633     __ scvtfws(v0, r0);
1634     break;
1635   case Bytecodes::_i2d:
1636     __ scvtfwd(v0, r0);
1637     break;
1638   case Bytecodes::_i2b:
1639     __ sxtbw(r0, r0);
1640     break;
1641   case Bytecodes::_i2c:
1642     __ uxthw(r0, r0);
1643     break;
1644   case Bytecodes::_i2s:
1645     __ sxthw(r0, r0);
1646     break;
1647   case Bytecodes::_l2i:
1648     __ uxtw(r0, r0);
1649     break;
1650   case Bytecodes::_l2f:
1651     __ scvtfs(v0, r0);
1652     break;
1653   case Bytecodes::_l2d:
1654     __ scvtfd(v0, r0);
1655     break;
1656   case Bytecodes::_f2i:
1657   {
1658     Label L_Okay;
1659     __ clear_fpsr();
1660     __ fcvtzsw(r0, v0);
1661     __ get_fpsr(r1);
1662     __ cbzw(r1, L_Okay);
1663     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2i),
1664                          0, 1, MacroAssembler::ret_type_integral);
1665     __ bind(L_Okay);
1666   }
1667     break;
1668   case Bytecodes::_f2l:
1669   {
1670     Label L_Okay;
1671     __ clear_fpsr();
1672     __ fcvtzs(r0, v0);
1673     __ get_fpsr(r1);
1674     __ cbzw(r1, L_Okay);
1675     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2l),
1676                          0, 1, MacroAssembler::ret_type_integral);
1677     __ bind(L_Okay);
1678   }
1679     break;
1680   case Bytecodes::_f2d:
1681     __ fcvts(v0, v0);
1682     break;
1683   case Bytecodes::_d2i:
1684   {
1685     Label L_Okay;
1686     __ clear_fpsr();
1687     __ fcvtzdw(r0, v0);
1688     __ get_fpsr(r1);
1689     __ cbzw(r1, L_Okay);
1690     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2i),
1691                          0, 1, MacroAssembler::ret_type_integral);
1692     __ bind(L_Okay);
1693   }
1694     break;
1695   case Bytecodes::_d2l:
1696   {
1697     Label L_Okay;
1698     __ clear_fpsr();
1699     __ fcvtzd(r0, v0);
1700     __ get_fpsr(r1);
1701     __ cbzw(r1, L_Okay);
1702     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2l),
1703                          0, 1, MacroAssembler::ret_type_integral);
1704     __ bind(L_Okay);
1705   }
1706     break;
1707   case Bytecodes::_d2f:
1708     __ fcvtd(v0, v0);
1709     break;
1710   default:
1711     ShouldNotReachHere();
1712   }
1713 }
1714 
1715 void TemplateTable::lcmp()
1716 {
1717   transition(ltos, itos);
1718   Label done;
1719   __ pop_l(r1);
1720   __ cmp(r1, r0);
1721   __ mov(r0, (u_int64_t)-1L);
1722   __ br(Assembler::LT, done);
1723   // __ mov(r0, 1UL);
1724   // __ csel(r0, r0, zr, Assembler::NE);
1725   // and here is a faster way
1726   __ csinc(r0, zr, zr, Assembler::EQ);
1727   __ bind(done);
1728 }
1729 
1730 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1731 {
1732   Label done;
1733   if (is_float) {
1734     // XXX get rid of pop here, use ... reg, mem32
1735     __ pop_f(v1);
1736     __ fcmps(v1, v0);
1737   } else {
1738     // XXX get rid of pop here, use ... reg, mem64
1739     __ pop_d(v1);
1740     __ fcmpd(v1, v0);
1741   }
1742   if (unordered_result < 0) {
1743     // we want -1 for unordered or less than, 0 for equal and 1 for
1744     // greater than.
1745     __ mov(r0, (u_int64_t)-1L);
1746     // for FP LT tests less than or unordered
1747     __ br(Assembler::LT, done);
1748     // install 0 for EQ otherwise 1
1749     __ csinc(r0, zr, zr, Assembler::EQ);
1750   } else {
1751     // we want -1 for less than, 0 for equal and 1 for unordered or
1752     // greater than.
1753     __ mov(r0, 1L);
1754     // for FP HI tests greater than or unordered
1755     __ br(Assembler::HI, done);
1756     // install 0 for EQ otherwise ~0
1757     __ csinv(r0, zr, zr, Assembler::EQ);
1758 
1759   }
1760   __ bind(done);
1761 }
1762 
1763 void TemplateTable::branch(bool is_jsr, bool is_wide)
1764 {
1765   // We might be moving to a safepoint.  The thread which calls
1766   // Interpreter::notice_safepoints() will effectively flush its cache
1767   // when it makes a system call, but we need to do something to
1768   // ensure that we see the changed dispatch table.
1769   __ membar(MacroAssembler::LoadLoad);
1770 
1771   __ profile_taken_branch(r0, r1);
1772   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1773                              InvocationCounter::counter_offset();
1774   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1775                               InvocationCounter::counter_offset();
1776 
1777   // load branch displacement
1778   if (!is_wide) {
1779     __ ldrh(r2, at_bcp(1));
1780     __ rev16(r2, r2);
1781     // sign extend the 16 bit value in r2
1782     __ sbfm(r2, r2, 0, 15);
1783   } else {
1784     __ ldrw(r2, at_bcp(1));
1785     __ revw(r2, r2);
1786     // sign extend the 32 bit value in r2
1787     __ sbfm(r2, r2, 0, 31);
1788   }
1789 
1790   // Handle all the JSR stuff here, then exit.
1791   // It's much shorter and cleaner than intermingling with the non-JSR
1792   // normal-branch stuff occurring below.
1793 
1794   if (is_jsr) {
1795     // Pre-load the next target bytecode into rscratch1
1796     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1797     // compute return address as bci
1798     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1799     __ add(rscratch2, rscratch2,
1800            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1801     __ sub(r1, rbcp, rscratch2);
1802     __ push_i(r1);
1803     // Adjust the bcp by the 16-bit displacement in r2
1804     __ add(rbcp, rbcp, r2);
1805     __ dispatch_only(vtos, /*generate_poll*/true);
1806     return;
1807   }
1808 
1809   // Normal (non-jsr) branch handling
1810 
1811   // Adjust the bcp by the displacement in r2
1812   __ add(rbcp, rbcp, r2);
1813 
1814   assert(UseLoopCounter || !UseOnStackReplacement,
1815          "on-stack-replacement requires loop counters");
1816   Label backedge_counter_overflow;
1817   Label profile_method;
1818   Label dispatch;
1819   if (UseLoopCounter) {
1820     // increment backedge counter for backward branches
1821     // r0: MDO
1822     // w1: MDO bumped taken-count
1823     // r2: target offset
1824     __ cmp(r2, zr);
1825     __ br(Assembler::GT, dispatch); // count only if backward branch
1826 
1827     // ECN: FIXME: This code smells
1828     // check if MethodCounters exists
1829     Label has_counters;
1830     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1831     __ cbnz(rscratch1, has_counters);
1832     __ push(r0);
1833     __ push(r1);
1834     __ push(r2);
1835     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1836             InterpreterRuntime::build_method_counters), rmethod);
1837     __ pop(r2);
1838     __ pop(r1);
1839     __ pop(r0);
1840     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1841     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1842     __ bind(has_counters);
1843 
1844     if (TieredCompilation) {
1845       Label no_mdo;
1846       int increment = InvocationCounter::count_increment;
1847       if (ProfileInterpreter) {
1848         // Are we profiling?
1849         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1850         __ cbz(r1, no_mdo);
1851         // Increment the MDO backedge counter
1852         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1853                                            in_bytes(InvocationCounter::counter_offset()));
1854         const Address mask(r1, in_bytes(MethodData::backedge_mask_offset()));
1855         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1856                                    r0, rscratch1, false, Assembler::EQ,
1857                                    UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1858         __ b(dispatch);
1859       }
1860       __ bind(no_mdo);
1861       // Increment backedge counter in MethodCounters*
1862       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1863       const Address mask(rscratch1, in_bytes(MethodCounters::backedge_mask_offset()));
1864       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1865                                  r0, rscratch2, false, Assembler::EQ,
1866                                  UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1867     } else { // not TieredCompilation
1868       // increment counter
1869       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1870       __ ldrw(r0, Address(rscratch2, be_offset));        // load backedge counter
1871       __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1872       __ strw(rscratch1, Address(rscratch2, be_offset));        // store counter
1873 
1874       __ ldrw(r0, Address(rscratch2, inv_offset));    // load invocation counter
1875       __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1876       __ addw(r0, r0, rscratch1);        // add both counters
1877 
1878       if (ProfileInterpreter) {
1879         // Test to see if we should create a method data oop
1880         __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_profile_limit_offset())));
1881         __ cmpw(r0, rscratch1);
1882         __ br(Assembler::LT, dispatch);
1883 
1884         // if no method data exists, go to profile method
1885         __ test_method_data_pointer(r0, profile_method);
1886 
1887         if (UseOnStackReplacement) {
1888           // check for overflow against w1 which is the MDO taken count
1889           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1890           __ cmpw(r1, rscratch1);
1891           __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1892 
1893           // When ProfileInterpreter is on, the backedge_count comes
1894           // from the MethodData*, which value does not get reset on
1895           // the call to frequency_counter_overflow().  To avoid
1896           // excessive calls to the overflow routine while the method is
1897           // being compiled, add a second test to make sure the overflow
1898           // function is called only once every overflow_frequency.
1899           const int overflow_frequency = 1024;
1900           __ andsw(r1, r1, overflow_frequency - 1);
1901           __ br(Assembler::EQ, backedge_counter_overflow);
1902 
1903         }
1904       } else {
1905         if (UseOnStackReplacement) {
1906           // check for overflow against w0, which is the sum of the
1907           // counters
1908           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1909           __ cmpw(r0, rscratch1);
1910           __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1911         }
1912       }
1913     }
1914     __ bind(dispatch);
1915   }
1916 
1917   // Pre-load the next target bytecode into rscratch1
1918   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1919 
1920   // continue with the bytecode @ target
1921   // rscratch1: target bytecode
1922   // rbcp: target bcp
1923   __ dispatch_only(vtos, /*generate_poll*/true);
1924 
1925   if (UseLoopCounter) {
1926     if (ProfileInterpreter) {
1927       // Out-of-line code to allocate method data oop.
1928       __ bind(profile_method);
1929       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1930       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1931       __ set_method_data_pointer_for_bcp();
1932       __ b(dispatch);
1933     }
1934 
1935     if (UseOnStackReplacement) {
1936       // invocation counter overflow
1937       __ bind(backedge_counter_overflow);
1938       __ neg(r2, r2);
1939       __ add(r2, r2, rbcp);     // branch bcp
1940       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1941       __ call_VM(noreg,
1942                  CAST_FROM_FN_PTR(address,
1943                                   InterpreterRuntime::frequency_counter_overflow),
1944                  r2);
1945       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1946 
1947       // r0: osr nmethod (osr ok) or NULL (osr not possible)
1948       // w1: target bytecode
1949       // r2: scratch
1950       __ cbz(r0, dispatch);     // test result -- no osr if null
1951       // nmethod may have been invalidated (VM may block upon call_VM return)
1952       __ ldrb(r2, Address(r0, nmethod::state_offset()));
1953       if (nmethod::in_use != 0)
1954         __ sub(r2, r2, nmethod::in_use);
1955       __ cbnz(r2, dispatch);
1956 
1957       // We have the address of an on stack replacement routine in r0
1958       // We need to prepare to execute the OSR method. First we must
1959       // migrate the locals and monitors off of the stack.
1960 
1961       __ mov(r19, r0);                             // save the nmethod
1962 
1963       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1964 
1965       // r0 is OSR buffer, move it to expected parameter location
1966       __ mov(j_rarg0, r0);
1967 
1968       // remove activation
1969       // get sender esp
1970       __ ldr(esp,
1971           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
1972       // remove frame anchor
1973       __ leave();
1974       // Ensure compiled code always sees stack at proper alignment
1975       __ andr(sp, esp, -16);
1976 
1977       // and begin the OSR nmethod
1978       __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
1979       __ br(rscratch1);
1980     }
1981   }
1982 }
1983 
1984 
1985 void TemplateTable::if_0cmp(Condition cc)
1986 {
1987   transition(itos, vtos);
1988   // assume branch is more often taken than not (loops use backward branches)
1989   Label not_taken;
1990   if (cc == equal)
1991     __ cbnzw(r0, not_taken);
1992   else if (cc == not_equal)
1993     __ cbzw(r0, not_taken);
1994   else {
1995     __ andsw(zr, r0, r0);
1996     __ br(j_not(cc), not_taken);
1997   }
1998 
1999   branch(false, false);
2000   __ bind(not_taken);
2001   __ profile_not_taken_branch(r0);
2002 }
2003 
2004 void TemplateTable::if_icmp(Condition cc)
2005 {
2006   transition(itos, vtos);
2007   // assume branch is more often taken than not (loops use backward branches)
2008   Label not_taken;
2009   __ pop_i(r1);
2010   __ cmpw(r1, r0, Assembler::LSL);
2011   __ br(j_not(cc), not_taken);
2012   branch(false, false);
2013   __ bind(not_taken);
2014   __ profile_not_taken_branch(r0);
2015 }
2016 
2017 void TemplateTable::if_nullcmp(Condition cc)
2018 {
2019   transition(atos, vtos);
2020   // assume branch is more often taken than not (loops use backward branches)
2021   Label not_taken;
2022   if (cc == equal)
2023     __ cbnz(r0, not_taken);
2024   else
2025     __ cbz(r0, not_taken);
2026   branch(false, false);
2027   __ bind(not_taken);
2028   __ profile_not_taken_branch(r0);
2029 }
2030 
2031 void TemplateTable::if_acmp(Condition cc)
2032 {
2033   transition(atos, vtos);
2034   // assume branch is more often taken than not (loops use backward branches)
2035   Label not_taken;
2036   __ pop_ptr(r1);
2037   __ cmp(r1, r0);
2038   __ br(j_not(cc), not_taken);
2039   branch(false, false);
2040   __ bind(not_taken);
2041   __ profile_not_taken_branch(r0);
2042 }
2043 
2044 void TemplateTable::ret() {
2045   transition(vtos, vtos);
2046   // We might be moving to a safepoint.  The thread which calls
2047   // Interpreter::notice_safepoints() will effectively flush its cache
2048   // when it makes a system call, but we need to do something to
2049   // ensure that we see the changed dispatch table.
2050   __ membar(MacroAssembler::LoadLoad);
2051 
2052   locals_index(r1);
2053   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2054   __ profile_ret(r1, r2);
2055   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2056   __ lea(rbcp, Address(rbcp, r1));
2057   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2058   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2059 }
2060 
2061 void TemplateTable::wide_ret() {
2062   transition(vtos, vtos);
2063   locals_index_wide(r1);
2064   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2065   __ profile_ret(r1, r2);
2066   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2067   __ lea(rbcp, Address(rbcp, r1));
2068   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2069   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2070 }
2071 
2072 
2073 void TemplateTable::tableswitch() {
2074   Label default_case, continue_execution;
2075   transition(itos, vtos);
2076   // align rbcp
2077   __ lea(r1, at_bcp(BytesPerInt));
2078   __ andr(r1, r1, -BytesPerInt);
2079   // load lo & hi
2080   __ ldrw(r2, Address(r1, BytesPerInt));
2081   __ ldrw(r3, Address(r1, 2 * BytesPerInt));
2082   __ rev32(r2, r2);
2083   __ rev32(r3, r3);
2084   // check against lo & hi
2085   __ cmpw(r0, r2);
2086   __ br(Assembler::LT, default_case);
2087   __ cmpw(r0, r3);
2088   __ br(Assembler::GT, default_case);
2089   // lookup dispatch offset
2090   __ subw(r0, r0, r2);
2091   __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2092   __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2093   __ profile_switch_case(r0, r1, r2);
2094   // continue execution
2095   __ bind(continue_execution);
2096   __ rev32(r3, r3);
2097   __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2098   __ add(rbcp, rbcp, r3, ext::sxtw);
2099   __ dispatch_only(vtos, /*generate_poll*/true);
2100   // handle default
2101   __ bind(default_case);
2102   __ profile_switch_default(r0);
2103   __ ldrw(r3, Address(r1, 0));
2104   __ b(continue_execution);
2105 }
2106 
2107 void TemplateTable::lookupswitch() {
2108   transition(itos, itos);
2109   __ stop("lookupswitch bytecode should have been rewritten");
2110 }
2111 
2112 void TemplateTable::fast_linearswitch() {
2113   transition(itos, vtos);
2114   Label loop_entry, loop, found, continue_execution;
2115   // bswap r0 so we can avoid bswapping the table entries
2116   __ rev32(r0, r0);
2117   // align rbcp
2118   __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2119                                     // this instruction (change offsets
2120                                     // below)
2121   __ andr(r19, r19, -BytesPerInt);
2122   // set counter
2123   __ ldrw(r1, Address(r19, BytesPerInt));
2124   __ rev32(r1, r1);
2125   __ b(loop_entry);
2126   // table search
2127   __ bind(loop);
2128   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2129   __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2130   __ cmpw(r0, rscratch1);
2131   __ br(Assembler::EQ, found);
2132   __ bind(loop_entry);
2133   __ subs(r1, r1, 1);
2134   __ br(Assembler::PL, loop);
2135   // default case
2136   __ profile_switch_default(r0);
2137   __ ldrw(r3, Address(r19, 0));
2138   __ b(continue_execution);
2139   // entry found -> get offset
2140   __ bind(found);
2141   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2142   __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2143   __ profile_switch_case(r1, r0, r19);
2144   // continue execution
2145   __ bind(continue_execution);
2146   __ rev32(r3, r3);
2147   __ add(rbcp, rbcp, r3, ext::sxtw);
2148   __ ldrb(rscratch1, Address(rbcp, 0));
2149   __ dispatch_only(vtos, /*generate_poll*/true);
2150 }
2151 
2152 void TemplateTable::fast_binaryswitch() {
2153   transition(itos, vtos);
2154   // Implementation using the following core algorithm:
2155   //
2156   // int binary_search(int key, LookupswitchPair* array, int n) {
2157   //   // Binary search according to "Methodik des Programmierens" by
2158   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2159   //   int i = 0;
2160   //   int j = n;
2161   //   while (i+1 < j) {
2162   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2163   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2164   //     // where a stands for the array and assuming that the (inexisting)
2165   //     // element a[n] is infinitely big.
2166   //     int h = (i + j) >> 1;
2167   //     // i < h < j
2168   //     if (key < array[h].fast_match()) {
2169   //       j = h;
2170   //     } else {
2171   //       i = h;
2172   //     }
2173   //   }
2174   //   // R: a[i] <= key < a[i+1] or Q
2175   //   // (i.e., if key is within array, i is the correct index)
2176   //   return i;
2177   // }
2178 
2179   // Register allocation
2180   const Register key   = r0; // already set (tosca)
2181   const Register array = r1;
2182   const Register i     = r2;
2183   const Register j     = r3;
2184   const Register h     = rscratch1;
2185   const Register temp  = rscratch2;
2186 
2187   // Find array start
2188   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2189                                           // get rid of this
2190                                           // instruction (change
2191                                           // offsets below)
2192   __ andr(array, array, -BytesPerInt);
2193 
2194   // Initialize i & j
2195   __ mov(i, 0);                            // i = 0;
2196   __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2197 
2198   // Convert j into native byteordering
2199   __ rev32(j, j);
2200 
2201   // And start
2202   Label entry;
2203   __ b(entry);
2204 
2205   // binary search loop
2206   {
2207     Label loop;
2208     __ bind(loop);
2209     // int h = (i + j) >> 1;
2210     __ addw(h, i, j);                           // h = i + j;
2211     __ lsrw(h, h, 1);                                   // h = (i + j) >> 1;
2212     // if (key < array[h].fast_match()) {
2213     //   j = h;
2214     // } else {
2215     //   i = h;
2216     // }
2217     // Convert array[h].match to native byte-ordering before compare
2218     __ ldr(temp, Address(array, h, Address::lsl(3)));
2219     __ rev32(temp, temp);
2220     __ cmpw(key, temp);
2221     // j = h if (key <  array[h].fast_match())
2222     __ csel(j, h, j, Assembler::LT);
2223     // i = h if (key >= array[h].fast_match())
2224     __ csel(i, h, i, Assembler::GE);
2225     // while (i+1 < j)
2226     __ bind(entry);
2227     __ addw(h, i, 1);          // i+1
2228     __ cmpw(h, j);             // i+1 < j
2229     __ br(Assembler::LT, loop);
2230   }
2231 
2232   // end of binary search, result index is i (must check again!)
2233   Label default_case;
2234   // Convert array[i].match to native byte-ordering before compare
2235   __ ldr(temp, Address(array, i, Address::lsl(3)));
2236   __ rev32(temp, temp);
2237   __ cmpw(key, temp);
2238   __ br(Assembler::NE, default_case);
2239 
2240   // entry found -> j = offset
2241   __ add(j, array, i, ext::uxtx, 3);
2242   __ ldrw(j, Address(j, BytesPerInt));
2243   __ profile_switch_case(i, key, array);
2244   __ rev32(j, j);
2245   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2246   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2247   __ dispatch_only(vtos, /*generate_poll*/true);
2248 
2249   // default case -> j = default offset
2250   __ bind(default_case);
2251   __ profile_switch_default(i);
2252   __ ldrw(j, Address(array, -2 * BytesPerInt));
2253   __ rev32(j, j);
2254   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2255   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2256   __ dispatch_only(vtos, /*generate_poll*/true);
2257 }
2258 
2259 
2260 void TemplateTable::_return(TosState state)
2261 {
2262   transition(state, state);
2263   assert(_desc->calls_vm(),
2264          "inconsistent calls_vm information"); // call in remove_activation
2265 
2266   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2267     assert(state == vtos, "only valid state");
2268 
2269     __ ldr(c_rarg1, aaddress(0));
2270     __ load_klass(r3, c_rarg1);
2271     __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2272     Label skip_register_finalizer;
2273     __ tbz(r3, exact_log2(JVM_ACC_HAS_FINALIZER), skip_register_finalizer);
2274 
2275     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2276 
2277     __ bind(skip_register_finalizer);
2278   }
2279 
2280   // Issue a StoreStore barrier after all stores but before return
2281   // from any constructor for any class with a final field.  We don't
2282   // know if this is a finalizer, so we always do so.
2283   if (_desc->bytecode() == Bytecodes::_return)
2284     __ membar(MacroAssembler::StoreStore);
2285 
2286   // Narrow result if state is itos but result type is smaller.
2287   // Need to narrow in the return bytecode rather than in generate_return_entry
2288   // since compiled code callers expect the result to already be narrowed.
2289   if (state == itos) {
2290     __ narrow(r0);
2291   }
2292 
2293   __ remove_activation(state);
2294   __ ret(lr);
2295 }
2296 
2297 // ----------------------------------------------------------------------------
2298 // Volatile variables demand their effects be made known to all CPU's
2299 // in order.  Store buffers on most chips allow reads & writes to
2300 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2301 // without some kind of memory barrier (i.e., it's not sufficient that
2302 // the interpreter does not reorder volatile references, the hardware
2303 // also must not reorder them).
2304 //
2305 // According to the new Java Memory Model (JMM):
2306 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2307 //     writes act as aquire & release, so:
2308 // (2) A read cannot let unrelated NON-volatile memory refs that
2309 //     happen after the read float up to before the read.  It's OK for
2310 //     non-volatile memory refs that happen before the volatile read to
2311 //     float down below it.
2312 // (3) Similar a volatile write cannot let unrelated NON-volatile
2313 //     memory refs that happen BEFORE the write float down to after the
2314 //     write.  It's OK for non-volatile memory refs that happen after the
2315 //     volatile write to float up before it.
2316 //
2317 // We only put in barriers around volatile refs (they are expensive),
2318 // not _between_ memory refs (that would require us to track the
2319 // flavor of the previous memory refs).  Requirements (2) and (3)
2320 // require some barriers before volatile stores and after volatile
2321 // loads.  These nearly cover requirement (1) but miss the
2322 // volatile-store-volatile-load case.  This final case is placed after
2323 // volatile-stores although it could just as well go before
2324 // volatile-loads.
2325 
2326 void TemplateTable::resolve_cache_and_index(int byte_no,
2327                                             Register Rcache,
2328                                             Register index,
2329                                             size_t index_size) {
2330   const Register temp = r19;
2331   assert_different_registers(Rcache, index, temp);
2332 
2333   Label resolved;
2334 
2335   Bytecodes::Code code = bytecode();
2336   switch (code) {
2337   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2338   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2339   }
2340 
2341   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2342   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2343   __ cmp(temp, (int) code);  // have we resolved this bytecode?
2344   __ br(Assembler::EQ, resolved);
2345 
2346   // resolve first time through
2347   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2348   __ mov(temp, (int) code);
2349   __ call_VM(noreg, entry, temp);
2350 
2351   // Update registers with resolved info
2352   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2353   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2354   // so all clients ofthis method must be modified accordingly
2355   __ bind(resolved);
2356 }
2357 
2358 // The Rcache and index registers must be set before call
2359 // n.b unlike x86 cache already includes the index offset
2360 void TemplateTable::load_field_cp_cache_entry(Register obj,
2361                                               Register cache,
2362                                               Register index,
2363                                               Register off,
2364                                               Register flags,
2365                                               bool is_static = false) {
2366   assert_different_registers(cache, index, flags, off);
2367 
2368   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2369   // Field offset
2370   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2371                                           ConstantPoolCacheEntry::f2_offset())));
2372   // Flags
2373   __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2374                                            ConstantPoolCacheEntry::flags_offset())));
2375 
2376   // klass overwrite register
2377   if (is_static) {
2378     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2379                                         ConstantPoolCacheEntry::f1_offset())));
2380     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2381     __ ldr(obj, Address(obj, mirror_offset));
2382     __ resolve_oop_handle(obj);
2383   }
2384 }
2385 
2386 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2387                                                Register method,
2388                                                Register itable_index,
2389                                                Register flags,
2390                                                bool is_invokevirtual,
2391                                                bool is_invokevfinal, /*unused*/
2392                                                bool is_invokedynamic) {
2393   // setup registers
2394   const Register cache = rscratch2;
2395   const Register index = r4;
2396   assert_different_registers(method, flags);
2397   assert_different_registers(method, cache, index);
2398   assert_different_registers(itable_index, flags);
2399   assert_different_registers(itable_index, cache, index);
2400   // determine constant pool cache field offsets
2401   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2402   const int method_offset = in_bytes(
2403     ConstantPoolCache::base_offset() +
2404       (is_invokevirtual
2405        ? ConstantPoolCacheEntry::f2_offset()
2406        : ConstantPoolCacheEntry::f1_offset()));
2407   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2408                                     ConstantPoolCacheEntry::flags_offset());
2409   // access constant pool cache fields
2410   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2411                                     ConstantPoolCacheEntry::f2_offset());
2412 
2413   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2414   resolve_cache_and_index(byte_no, cache, index, index_size);
2415   __ ldr(method, Address(cache, method_offset));
2416 
2417   if (itable_index != noreg) {
2418     __ ldr(itable_index, Address(cache, index_offset));
2419   }
2420   __ ldrw(flags, Address(cache, flags_offset));
2421 }
2422 
2423 
2424 // The registers cache and index expected to be set before call.
2425 // Correct values of the cache and index registers are preserved.
2426 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2427                                             bool is_static, bool has_tos) {
2428   // do the JVMTI work here to avoid disturbing the register state below
2429   // We use c_rarg registers here because we want to use the register used in
2430   // the call to the VM
2431   if (JvmtiExport::can_post_field_access()) {
2432     // Check to see if a field access watch has been set before we
2433     // take the time to call into the VM.
2434     Label L1;
2435     assert_different_registers(cache, index, r0);
2436     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2437     __ ldrw(r0, Address(rscratch1));
2438     __ cbzw(r0, L1);
2439 
2440     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2441     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2442 
2443     if (is_static) {
2444       __ mov(c_rarg1, zr); // NULL object reference
2445     } else {
2446       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2447       __ verify_oop(c_rarg1);
2448     }
2449     // c_rarg1: object pointer or NULL
2450     // c_rarg2: cache entry pointer
2451     // c_rarg3: jvalue object on the stack
2452     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2453                                        InterpreterRuntime::post_field_access),
2454                c_rarg1, c_rarg2, c_rarg3);
2455     __ get_cache_and_index_at_bcp(cache, index, 1);
2456     __ bind(L1);
2457   }
2458 }
2459 
2460 void TemplateTable::pop_and_check_object(Register r)
2461 {
2462   __ pop_ptr(r);
2463   __ null_check(r);  // for field access must check obj.
2464   __ verify_oop(r);
2465 }
2466 
2467 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2468 {
2469   const Register cache = r2;
2470   const Register index = r3;
2471   const Register obj   = r4;
2472   const Register off   = r19;
2473   const Register flags = r0;
2474   const Register raw_flags = r6;
2475   const Register bc    = r4; // uses same reg as obj, so don't mix them
2476 
2477   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2478   jvmti_post_field_access(cache, index, is_static, false);
2479   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2480 
2481   if (!is_static) {
2482     // obj is on the stack
2483     pop_and_check_object(obj);
2484   }
2485 
2486   // 8179954: We need to make sure that the code generated for
2487   // volatile accesses forms a sequentially-consistent set of
2488   // operations when combined with STLR and LDAR.  Without a leading
2489   // membar it's possible for a simple Dekker test to fail if loads
2490   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2491   // the stores in one method and we interpret the loads in another.
2492   if (! UseBarriersForVolatile) {
2493     Label notVolatile;
2494     __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2495     __ membar(MacroAssembler::AnyAny);
2496     __ bind(notVolatile);
2497   }
2498 
2499   const Address field(obj, off);
2500 
2501   Label Done, notByte, notBool, notInt, notShort, notChar,
2502               notLong, notFloat, notObj, notDouble;
2503 
2504   // x86 uses a shift and mask or wings it with a shift plus assert
2505   // the mask is not needed. aarch64 just uses bitfield extract
2506   __ ubfxw(flags, raw_flags, ConstantPoolCacheEntry::tos_state_shift,
2507            ConstantPoolCacheEntry::tos_state_bits);
2508 
2509   assert(btos == 0, "change code, btos != 0");
2510   __ cbnz(flags, notByte);
2511 
2512   // Don't rewrite getstatic, only getfield
2513   if (is_static) rc = may_not_rewrite;
2514 
2515   // btos
2516   __ load_signed_byte(r0, field);
2517   __ push(btos);
2518   // Rewrite bytecode to be faster
2519   if (rc == may_rewrite) {
2520     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2521   }
2522   __ b(Done);
2523 
2524   __ bind(notByte);
2525   __ cmp(flags, ztos);
2526   __ br(Assembler::NE, notBool);
2527 
2528   // ztos (same code as btos)
2529   __ ldrsb(r0, field);
2530   __ push(ztos);
2531   // Rewrite bytecode to be faster
2532   if (rc == may_rewrite) {
2533     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2534     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2535   }
2536   __ b(Done);
2537 
2538   __ bind(notBool);
2539   __ cmp(flags, atos);
2540   __ br(Assembler::NE, notObj);
2541   // atos
2542   do_oop_load(_masm, field, r0, IN_HEAP);
2543   __ push(atos);
2544   if (rc == may_rewrite) {
2545     patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2546   }
2547   __ b(Done);
2548 
2549   __ bind(notObj);
2550   __ cmp(flags, itos);
2551   __ br(Assembler::NE, notInt);
2552   // itos
2553   __ ldrw(r0, field);
2554   __ push(itos);
2555   // Rewrite bytecode to be faster
2556   if (rc == may_rewrite) {
2557     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2558   }
2559   __ b(Done);
2560 
2561   __ bind(notInt);
2562   __ cmp(flags, ctos);
2563   __ br(Assembler::NE, notChar);
2564   // ctos
2565   __ load_unsigned_short(r0, field);
2566   __ push(ctos);
2567   // Rewrite bytecode to be faster
2568   if (rc == may_rewrite) {
2569     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2570   }
2571   __ b(Done);
2572 
2573   __ bind(notChar);
2574   __ cmp(flags, stos);
2575   __ br(Assembler::NE, notShort);
2576   // stos
2577   __ load_signed_short(r0, field);
2578   __ push(stos);
2579   // Rewrite bytecode to be faster
2580   if (rc == may_rewrite) {
2581     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2582   }
2583   __ b(Done);
2584 
2585   __ bind(notShort);
2586   __ cmp(flags, ltos);
2587   __ br(Assembler::NE, notLong);
2588   // ltos
2589   __ ldr(r0, field);
2590   __ push(ltos);
2591   // Rewrite bytecode to be faster
2592   if (rc == may_rewrite) {
2593     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2594   }
2595   __ b(Done);
2596 
2597   __ bind(notLong);
2598   __ cmp(flags, ftos);
2599   __ br(Assembler::NE, notFloat);
2600   // ftos
2601   __ ldrs(v0, field);
2602   __ push(ftos);
2603   // Rewrite bytecode to be faster
2604   if (rc == may_rewrite) {
2605     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2606   }
2607   __ b(Done);
2608 
2609   __ bind(notFloat);
2610 #ifdef ASSERT
2611   __ cmp(flags, dtos);
2612   __ br(Assembler::NE, notDouble);
2613 #endif
2614   // dtos
2615   __ ldrd(v0, field);
2616   __ push(dtos);
2617   // Rewrite bytecode to be faster
2618   if (rc == may_rewrite) {
2619     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2620   }
2621 #ifdef ASSERT
2622   __ b(Done);
2623 
2624   __ bind(notDouble);
2625   __ stop("Bad state");
2626 #endif
2627 
2628   __ bind(Done);
2629 
2630   Label notVolatile;
2631   __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2632   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2633   __ bind(notVolatile);
2634 }
2635 
2636 
2637 void TemplateTable::getfield(int byte_no)
2638 {
2639   getfield_or_static(byte_no, false);
2640 }
2641 
2642 void TemplateTable::nofast_getfield(int byte_no) {
2643   getfield_or_static(byte_no, false, may_not_rewrite);
2644 }
2645 
2646 void TemplateTable::getstatic(int byte_no)
2647 {
2648   getfield_or_static(byte_no, true);
2649 }
2650 
2651 // The registers cache and index expected to be set before call.
2652 // The function may destroy various registers, just not the cache and index registers.
2653 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2654   transition(vtos, vtos);
2655 
2656   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2657 
2658   if (JvmtiExport::can_post_field_modification()) {
2659     // Check to see if a field modification watch has been set before
2660     // we take the time to call into the VM.
2661     Label L1;
2662     assert_different_registers(cache, index, r0);
2663     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2664     __ ldrw(r0, Address(rscratch1));
2665     __ cbz(r0, L1);
2666 
2667     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2668 
2669     if (is_static) {
2670       // Life is simple.  Null out the object pointer.
2671       __ mov(c_rarg1, zr);
2672     } else {
2673       // Life is harder. The stack holds the value on top, followed by
2674       // the object.  We don't know the size of the value, though; it
2675       // could be one or two words depending on its type. As a result,
2676       // we must find the type to determine where the object is.
2677       __ ldrw(c_rarg3, Address(c_rarg2,
2678                                in_bytes(cp_base_offset +
2679                                         ConstantPoolCacheEntry::flags_offset())));
2680       __ lsr(c_rarg3, c_rarg3,
2681              ConstantPoolCacheEntry::tos_state_shift);
2682       ConstantPoolCacheEntry::verify_tos_state_shift();
2683       Label nope2, done, ok;
2684       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2685       __ cmpw(c_rarg3, ltos);
2686       __ br(Assembler::EQ, ok);
2687       __ cmpw(c_rarg3, dtos);
2688       __ br(Assembler::NE, nope2);
2689       __ bind(ok);
2690       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2691       __ bind(nope2);
2692     }
2693     // cache entry pointer
2694     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2695     // object (tos)
2696     __ mov(c_rarg3, esp);
2697     // c_rarg1: object pointer set up above (NULL if static)
2698     // c_rarg2: cache entry pointer
2699     // c_rarg3: jvalue object on the stack
2700     __ call_VM(noreg,
2701                CAST_FROM_FN_PTR(address,
2702                                 InterpreterRuntime::post_field_modification),
2703                c_rarg1, c_rarg2, c_rarg3);
2704     __ get_cache_and_index_at_bcp(cache, index, 1);
2705     __ bind(L1);
2706   }
2707 }
2708 
2709 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2710   transition(vtos, vtos);
2711 
2712   const Register cache = r2;
2713   const Register index = r3;
2714   const Register obj   = r2;
2715   const Register off   = r19;
2716   const Register flags = r0;
2717   const Register bc    = r4;
2718 
2719   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2720   jvmti_post_field_mod(cache, index, is_static);
2721   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2722 
2723   Label Done;
2724   __ mov(r5, flags);
2725 
2726   {
2727     Label notVolatile;
2728     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2729     __ membar(MacroAssembler::StoreStore);
2730     __ bind(notVolatile);
2731   }
2732 
2733   // field address
2734   const Address field(obj, off);
2735 
2736   Label notByte, notBool, notInt, notShort, notChar,
2737         notLong, notFloat, notObj, notDouble;
2738 
2739   // x86 uses a shift and mask or wings it with a shift plus assert
2740   // the mask is not needed. aarch64 just uses bitfield extract
2741   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2742 
2743   assert(btos == 0, "change code, btos != 0");
2744   __ cbnz(flags, notByte);
2745 
2746   // Don't rewrite putstatic, only putfield
2747   if (is_static) rc = may_not_rewrite;
2748 
2749   // btos
2750   {
2751     __ pop(btos);
2752     if (!is_static) pop_and_check_object(obj);
2753     __ strb(r0, field);
2754     if (rc == may_rewrite) {
2755       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2756     }
2757     __ b(Done);
2758   }
2759 
2760   __ bind(notByte);
2761   __ cmp(flags, ztos);
2762   __ br(Assembler::NE, notBool);
2763 
2764   // ztos
2765   {
2766     __ pop(ztos);
2767     if (!is_static) pop_and_check_object(obj);
2768     __ andw(r0, r0, 0x1);
2769     __ strb(r0, field);
2770     if (rc == may_rewrite) {
2771       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2772     }
2773     __ b(Done);
2774   }
2775 
2776   __ bind(notBool);
2777   __ cmp(flags, atos);
2778   __ br(Assembler::NE, notObj);
2779 
2780   // atos
2781   {
2782     __ pop(atos);
2783     if (!is_static) pop_and_check_object(obj);
2784     // Store into the field
2785     do_oop_store(_masm, field, r0, IN_HEAP);
2786     if (rc == may_rewrite) {
2787       patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2788     }
2789     __ b(Done);
2790   }
2791 
2792   __ bind(notObj);
2793   __ cmp(flags, itos);
2794   __ br(Assembler::NE, notInt);
2795 
2796   // itos
2797   {
2798     __ pop(itos);
2799     if (!is_static) pop_and_check_object(obj);
2800     __ strw(r0, field);
2801     if (rc == may_rewrite) {
2802       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
2803     }
2804     __ b(Done);
2805   }
2806 
2807   __ bind(notInt);
2808   __ cmp(flags, ctos);
2809   __ br(Assembler::NE, notChar);
2810 
2811   // ctos
2812   {
2813     __ pop(ctos);
2814     if (!is_static) pop_and_check_object(obj);
2815     __ strh(r0, field);
2816     if (rc == may_rewrite) {
2817       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
2818     }
2819     __ b(Done);
2820   }
2821 
2822   __ bind(notChar);
2823   __ cmp(flags, stos);
2824   __ br(Assembler::NE, notShort);
2825 
2826   // stos
2827   {
2828     __ pop(stos);
2829     if (!is_static) pop_and_check_object(obj);
2830     __ strh(r0, field);
2831     if (rc == may_rewrite) {
2832       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
2833     }
2834     __ b(Done);
2835   }
2836 
2837   __ bind(notShort);
2838   __ cmp(flags, ltos);
2839   __ br(Assembler::NE, notLong);
2840 
2841   // ltos
2842   {
2843     __ pop(ltos);
2844     if (!is_static) pop_and_check_object(obj);
2845     __ str(r0, field);
2846     if (rc == may_rewrite) {
2847       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
2848     }
2849     __ b(Done);
2850   }
2851 
2852   __ bind(notLong);
2853   __ cmp(flags, ftos);
2854   __ br(Assembler::NE, notFloat);
2855 
2856   // ftos
2857   {
2858     __ pop(ftos);
2859     if (!is_static) pop_and_check_object(obj);
2860     __ strs(v0, field);
2861     if (rc == may_rewrite) {
2862       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
2863     }
2864     __ b(Done);
2865   }
2866 
2867   __ bind(notFloat);
2868 #ifdef ASSERT
2869   __ cmp(flags, dtos);
2870   __ br(Assembler::NE, notDouble);
2871 #endif
2872 
2873   // dtos
2874   {
2875     __ pop(dtos);
2876     if (!is_static) pop_and_check_object(obj);
2877     __ strd(v0, field);
2878     if (rc == may_rewrite) {
2879       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
2880     }
2881   }
2882 
2883 #ifdef ASSERT
2884   __ b(Done);
2885 
2886   __ bind(notDouble);
2887   __ stop("Bad state");
2888 #endif
2889 
2890   __ bind(Done);
2891 
2892   {
2893     Label notVolatile;
2894     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2895     __ membar(MacroAssembler::StoreLoad);
2896     __ bind(notVolatile);
2897   }
2898 }
2899 
2900 void TemplateTable::putfield(int byte_no)
2901 {
2902   putfield_or_static(byte_no, false);
2903 }
2904 
2905 void TemplateTable::nofast_putfield(int byte_no) {
2906   putfield_or_static(byte_no, false, may_not_rewrite);
2907 }
2908 
2909 void TemplateTable::putstatic(int byte_no) {
2910   putfield_or_static(byte_no, true);
2911 }
2912 
2913 void TemplateTable::jvmti_post_fast_field_mod()
2914 {
2915   if (JvmtiExport::can_post_field_modification()) {
2916     // Check to see if a field modification watch has been set before
2917     // we take the time to call into the VM.
2918     Label L2;
2919     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2920     __ ldrw(c_rarg3, Address(rscratch1));
2921     __ cbzw(c_rarg3, L2);
2922     __ pop_ptr(r19);                  // copy the object pointer from tos
2923     __ verify_oop(r19);
2924     __ push_ptr(r19);                 // put the object pointer back on tos
2925     // Save tos values before call_VM() clobbers them. Since we have
2926     // to do it for every data type, we use the saved values as the
2927     // jvalue object.
2928     switch (bytecode()) {          // load values into the jvalue object
2929     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
2930     case Bytecodes::_fast_bputfield: // fall through
2931     case Bytecodes::_fast_zputfield: // fall through
2932     case Bytecodes::_fast_sputfield: // fall through
2933     case Bytecodes::_fast_cputfield: // fall through
2934     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
2935     case Bytecodes::_fast_dputfield: __ push_d(); break;
2936     case Bytecodes::_fast_fputfield: __ push_f(); break;
2937     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
2938 
2939     default:
2940       ShouldNotReachHere();
2941     }
2942     __ mov(c_rarg3, esp);             // points to jvalue on the stack
2943     // access constant pool cache entry
2944     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
2945     __ verify_oop(r19);
2946     // r19: object pointer copied above
2947     // c_rarg2: cache entry pointer
2948     // c_rarg3: jvalue object on the stack
2949     __ call_VM(noreg,
2950                CAST_FROM_FN_PTR(address,
2951                                 InterpreterRuntime::post_field_modification),
2952                r19, c_rarg2, c_rarg3);
2953 
2954     switch (bytecode()) {             // restore tos values
2955     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
2956     case Bytecodes::_fast_bputfield: // fall through
2957     case Bytecodes::_fast_zputfield: // fall through
2958     case Bytecodes::_fast_sputfield: // fall through
2959     case Bytecodes::_fast_cputfield: // fall through
2960     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
2961     case Bytecodes::_fast_dputfield: __ pop_d(); break;
2962     case Bytecodes::_fast_fputfield: __ pop_f(); break;
2963     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
2964     }
2965     __ bind(L2);
2966   }
2967 }
2968 
2969 void TemplateTable::fast_storefield(TosState state)
2970 {
2971   transition(state, vtos);
2972 
2973   ByteSize base = ConstantPoolCache::base_offset();
2974 
2975   jvmti_post_fast_field_mod();
2976 
2977   // access constant pool cache
2978   __ get_cache_and_index_at_bcp(r2, r1, 1);
2979 
2980   // test for volatile with r3
2981   __ ldrw(r3, Address(r2, in_bytes(base +
2982                                    ConstantPoolCacheEntry::flags_offset())));
2983 
2984   // replace index with field offset from cache entry
2985   __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
2986 
2987   {
2988     Label notVolatile;
2989     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2990     __ membar(MacroAssembler::StoreStore);
2991     __ bind(notVolatile);
2992   }
2993 
2994   Label notVolatile;
2995 
2996   // Get object from stack
2997   pop_and_check_object(r2);
2998 
2999   // field address
3000   const Address field(r2, r1);
3001 
3002   // access field
3003   switch (bytecode()) {
3004   case Bytecodes::_fast_aputfield:
3005     do_oop_store(_masm, field, r0, IN_HEAP);
3006     break;
3007   case Bytecodes::_fast_lputfield:
3008     __ str(r0, field);
3009     break;
3010   case Bytecodes::_fast_iputfield:
3011     __ strw(r0, field);
3012     break;
3013   case Bytecodes::_fast_zputfield:
3014     __ andw(r0, r0, 0x1);  // boolean is true if LSB is 1
3015     // fall through to bputfield
3016   case Bytecodes::_fast_bputfield:
3017     __ strb(r0, field);
3018     break;
3019   case Bytecodes::_fast_sputfield:
3020     // fall through
3021   case Bytecodes::_fast_cputfield:
3022     __ strh(r0, field);
3023     break;
3024   case Bytecodes::_fast_fputfield:
3025     __ strs(v0, field);
3026     break;
3027   case Bytecodes::_fast_dputfield:
3028     __ strd(v0, field);
3029     break;
3030   default:
3031     ShouldNotReachHere();
3032   }
3033 
3034   {
3035     Label notVolatile;
3036     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3037     __ membar(MacroAssembler::StoreLoad);
3038     __ bind(notVolatile);
3039   }
3040 }
3041 
3042 
3043 void TemplateTable::fast_accessfield(TosState state)
3044 {
3045   transition(atos, state);
3046   // Do the JVMTI work here to avoid disturbing the register state below
3047   if (JvmtiExport::can_post_field_access()) {
3048     // Check to see if a field access watch has been set before we
3049     // take the time to call into the VM.
3050     Label L1;
3051     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3052     __ ldrw(r2, Address(rscratch1));
3053     __ cbzw(r2, L1);
3054     // access constant pool cache entry
3055     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
3056     __ verify_oop(r0);
3057     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
3058     __ mov(c_rarg1, r0);
3059     // c_rarg1: object pointer copied above
3060     // c_rarg2: cache entry pointer
3061     __ call_VM(noreg,
3062                CAST_FROM_FN_PTR(address,
3063                                 InterpreterRuntime::post_field_access),
3064                c_rarg1, c_rarg2);
3065     __ pop_ptr(r0); // restore object pointer
3066     __ bind(L1);
3067   }
3068 
3069   // access constant pool cache
3070   __ get_cache_and_index_at_bcp(r2, r1, 1);
3071   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3072                                   ConstantPoolCacheEntry::f2_offset())));
3073   __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3074                                    ConstantPoolCacheEntry::flags_offset())));
3075 
3076   // r0: object
3077   __ verify_oop(r0);
3078   __ null_check(r0);
3079   const Address field(r0, r1);
3080 
3081   // 8179954: We need to make sure that the code generated for
3082   // volatile accesses forms a sequentially-consistent set of
3083   // operations when combined with STLR and LDAR.  Without a leading
3084   // membar it's possible for a simple Dekker test to fail if loads
3085   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3086   // the stores in one method and we interpret the loads in another.
3087   if (! UseBarriersForVolatile) {
3088     Label notVolatile;
3089     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3090     __ membar(MacroAssembler::AnyAny);
3091     __ bind(notVolatile);
3092   }
3093 
3094   // access field
3095   switch (bytecode()) {
3096   case Bytecodes::_fast_agetfield:
3097     do_oop_load(_masm, field, r0, IN_HEAP);
3098     __ verify_oop(r0);
3099     break;
3100   case Bytecodes::_fast_lgetfield:
3101     __ ldr(r0, field);
3102     break;
3103   case Bytecodes::_fast_igetfield:
3104     __ ldrw(r0, field);
3105     break;
3106   case Bytecodes::_fast_bgetfield:
3107     __ load_signed_byte(r0, field);
3108     break;
3109   case Bytecodes::_fast_sgetfield:
3110     __ load_signed_short(r0, field);
3111     break;
3112   case Bytecodes::_fast_cgetfield:
3113     __ load_unsigned_short(r0, field);
3114     break;
3115   case Bytecodes::_fast_fgetfield:
3116     __ ldrs(v0, field);
3117     break;
3118   case Bytecodes::_fast_dgetfield:
3119     __ ldrd(v0, field);
3120     break;
3121   default:
3122     ShouldNotReachHere();
3123   }
3124   {
3125     Label notVolatile;
3126     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3127     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3128     __ bind(notVolatile);
3129   }
3130 }
3131 
3132 void TemplateTable::fast_xaccess(TosState state)
3133 {
3134   transition(vtos, state);
3135 
3136   // get receiver
3137   __ ldr(r0, aaddress(0));
3138   // access constant pool cache
3139   __ get_cache_and_index_at_bcp(r2, r3, 2);
3140   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3141                                   ConstantPoolCacheEntry::f2_offset())));
3142 
3143   // 8179954: We need to make sure that the code generated for
3144   // volatile accesses forms a sequentially-consistent set of
3145   // operations when combined with STLR and LDAR.  Without a leading
3146   // membar it's possible for a simple Dekker test to fail if loads
3147   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3148   // the stores in one method and we interpret the loads in another.
3149   if (! UseBarriersForVolatile) {
3150     Label notVolatile;
3151     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3152                                      ConstantPoolCacheEntry::flags_offset())));
3153     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3154     __ membar(MacroAssembler::AnyAny);
3155     __ bind(notVolatile);
3156   }
3157 
3158   // make sure exception is reported in correct bcp range (getfield is
3159   // next instruction)
3160   __ increment(rbcp);
3161   __ null_check(r0);
3162   switch (state) {
3163   case itos:
3164     __ ldrw(r0, Address(r0, r1, Address::lsl(0)));
3165     break;
3166   case atos:
3167     do_oop_load(_masm, Address(r0, r1, Address::lsl(0)), r0, IN_HEAP);
3168     __ verify_oop(r0);
3169     break;
3170   case ftos:
3171     __ ldrs(v0, Address(r0, r1, Address::lsl(0)));
3172     break;
3173   default:
3174     ShouldNotReachHere();
3175   }
3176 
3177   {
3178     Label notVolatile;
3179     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3180                                      ConstantPoolCacheEntry::flags_offset())));
3181     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3182     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3183     __ bind(notVolatile);
3184   }
3185 
3186   __ decrement(rbcp);
3187 }
3188 
3189 
3190 
3191 //-----------------------------------------------------------------------------
3192 // Calls
3193 
3194 void TemplateTable::count_calls(Register method, Register temp)
3195 {
3196   __ call_Unimplemented();
3197 }
3198 
3199 void TemplateTable::prepare_invoke(int byte_no,
3200                                    Register method, // linked method (or i-klass)
3201                                    Register index,  // itable index, MethodType, etc.
3202                                    Register recv,   // if caller wants to see it
3203                                    Register flags   // if caller wants to test it
3204                                    ) {
3205   // determine flags
3206   Bytecodes::Code code = bytecode();
3207   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3208   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3209   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3210   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3211   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3212   const bool load_receiver       = (recv  != noreg);
3213   const bool save_flags          = (flags != noreg);
3214   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3215   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3216   assert(flags == noreg || flags == r3, "");
3217   assert(recv  == noreg || recv  == r2, "");
3218 
3219   // setup registers & access constant pool cache
3220   if (recv  == noreg)  recv  = r2;
3221   if (flags == noreg)  flags = r3;
3222   assert_different_registers(method, index, recv, flags);
3223 
3224   // save 'interpreter return address'
3225   __ save_bcp();
3226 
3227   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3228 
3229   // maybe push appendix to arguments (just before return address)
3230   if (is_invokedynamic || is_invokehandle) {
3231     Label L_no_push;
3232     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3233     // Push the appendix as a trailing parameter.
3234     // This must be done before we get the receiver,
3235     // since the parameter_size includes it.
3236     __ push(r19);
3237     __ mov(r19, index);
3238     assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
3239     __ load_resolved_reference_at_index(index, r19);
3240     __ pop(r19);
3241     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3242     __ bind(L_no_push);
3243   }
3244 
3245   // load receiver if needed (note: no return address pushed yet)
3246   if (load_receiver) {
3247     __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3248     // FIXME -- is this actually correct? looks like it should be 2
3249     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3250     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3251     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3252     // __ movptr(recv, recv_addr);
3253     __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3254     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3255     __ verify_oop(recv);
3256   }
3257 
3258   // compute return type
3259   // x86 uses a shift and mask or wings it with a shift plus assert
3260   // the mask is not needed. aarch64 just uses bitfield extract
3261   __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3262   // load return address
3263   {
3264     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3265     __ mov(rscratch1, table_addr);
3266     __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3267   }
3268 }
3269 
3270 
3271 void TemplateTable::invokevirtual_helper(Register index,
3272                                          Register recv,
3273                                          Register flags)
3274 {
3275   // Uses temporary registers r0, r3
3276   assert_different_registers(index, recv, r0, r3);
3277   // Test for an invoke of a final method
3278   Label notFinal;
3279   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3280 
3281   const Register method = index;  // method must be rmethod
3282   assert(method == rmethod,
3283          "methodOop must be rmethod for interpreter calling convention");
3284 
3285   // do the call - the index is actually the method to call
3286   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3287 
3288   // It's final, need a null check here!
3289   __ null_check(recv);
3290 
3291   // profile this call
3292   __ profile_final_call(r0);
3293   __ profile_arguments_type(r0, method, r4, true);
3294 
3295   __ jump_from_interpreted(method, r0);
3296 
3297   __ bind(notFinal);
3298 
3299   // get receiver klass
3300   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3301   __ load_klass(r0, recv);
3302 
3303   // profile this call
3304   __ profile_virtual_call(r0, rlocals, r3);
3305 
3306   // get target methodOop & entry point
3307   __ lookup_virtual_method(r0, index, method);
3308   __ profile_arguments_type(r3, method, r4, true);
3309   // FIXME -- this looks completely redundant. is it?
3310   // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3311   __ jump_from_interpreted(method, r3);
3312 }
3313 
3314 void TemplateTable::invokevirtual(int byte_no)
3315 {
3316   transition(vtos, vtos);
3317   assert(byte_no == f2_byte, "use this argument");
3318 
3319   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3320 
3321   // rmethod: index (actually a Method*)
3322   // r2: receiver
3323   // r3: flags
3324 
3325   invokevirtual_helper(rmethod, r2, r3);
3326 }
3327 
3328 void TemplateTable::invokespecial(int byte_no)
3329 {
3330   transition(vtos, vtos);
3331   assert(byte_no == f1_byte, "use this argument");
3332 
3333   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3334                  r2);  // get receiver also for null check
3335   __ verify_oop(r2);
3336   __ null_check(r2);
3337   // do the call
3338   __ profile_call(r0);
3339   __ profile_arguments_type(r0, rmethod, rbcp, false);
3340   __ jump_from_interpreted(rmethod, r0);
3341 }
3342 
3343 void TemplateTable::invokestatic(int byte_no)
3344 {
3345   transition(vtos, vtos);
3346   assert(byte_no == f1_byte, "use this argument");
3347 
3348   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3349   // do the call
3350   __ profile_call(r0);
3351   __ profile_arguments_type(r0, rmethod, r4, false);
3352   __ jump_from_interpreted(rmethod, r0);
3353 }
3354 
3355 void TemplateTable::fast_invokevfinal(int byte_no)
3356 {
3357   __ call_Unimplemented();
3358 }
3359 
3360 void TemplateTable::invokeinterface(int byte_no) {
3361   transition(vtos, vtos);
3362   assert(byte_no == f1_byte, "use this argument");
3363 
3364   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 Method*
3365                  r2, r3); // recv, flags
3366 
3367   // r0: interface klass (from f1)
3368   // rmethod: method (from f2)
3369   // r2: receiver
3370   // r3: flags
3371 
3372   // Special case of invokeinterface called for virtual method of
3373   // java.lang.Object.  See cpCacheOop.cpp for details.
3374   // This code isn't produced by javac, but could be produced by
3375   // another compliant java compiler.
3376   Label notMethod;
3377   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notMethod);
3378 
3379   invokevirtual_helper(rmethod, r2, r3);
3380   __ bind(notMethod);
3381 
3382   // Get receiver klass into r3 - also a null check
3383   __ restore_locals();
3384   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3385   __ load_klass(r3, r2);
3386 
3387   Label no_such_interface, no_such_method;
3388 
3389   // Preserve method for throw_AbstractMethodErrorVerbose.
3390   __ mov(r16, rmethod);
3391   // Receiver subtype check against REFC.
3392   // Superklass in r0. Subklass in r3. Blows rscratch2, r13
3393   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3394                              r3, r0, noreg,
3395                              // outputs: scan temp. reg, scan temp. reg
3396                              rscratch2, r13,
3397                              no_such_interface,
3398                              /*return_method=*/false);
3399 
3400   // profile this call
3401   __ profile_virtual_call(r3, r13, r19);
3402 
3403   // Get declaring interface class from method, and itable index
3404   __ ldr(r0, Address(rmethod, Method::const_offset()));
3405   __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
3406   __ ldr(r0, Address(r0, ConstantPool::pool_holder_offset_in_bytes()));
3407   __ ldrw(rmethod, Address(rmethod, Method::itable_index_offset()));
3408   __ subw(rmethod, rmethod, Method::itable_index_max);
3409   __ negw(rmethod, rmethod);
3410 
3411   // Preserve recvKlass for throw_AbstractMethodErrorVerbose.
3412   __ mov(rlocals, r3);
3413   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3414                              rlocals, r0, rmethod,
3415                              // outputs: method, scan temp. reg
3416                              rmethod, r13,
3417                              no_such_interface);
3418 
3419   // rmethod,: methodOop to call
3420   // r2: receiver
3421   // Check for abstract method error
3422   // Note: This should be done more efficiently via a throw_abstract_method_error
3423   //       interpreter entry point and a conditional jump to it in case of a null
3424   //       method.
3425   __ cbz(rmethod, no_such_method);
3426 
3427   __ profile_arguments_type(r3, rmethod, r13, true);
3428 
3429   // do the call
3430   // r2: receiver
3431   // rmethod,: methodOop
3432   __ jump_from_interpreted(rmethod, r3);
3433   __ should_not_reach_here();
3434 
3435   // exception handling code follows...
3436   // note: must restore interpreter registers to canonical
3437   //       state for exception handling to work correctly!
3438 
3439   __ bind(no_such_method);
3440   // throw exception
3441   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3442   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3443   // Pass arguments for generating a verbose error message.
3444   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose), r3, r16);
3445   // the call_VM checks for exception, so we should never return here.
3446   __ should_not_reach_here();
3447 
3448   __ bind(no_such_interface);
3449   // throw exception
3450   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3451   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3452   // Pass arguments for generating a verbose error message.
3453   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3454                    InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose), r3, r0);
3455   // the call_VM checks for exception, so we should never return here.
3456   __ should_not_reach_here();
3457   return;
3458 }
3459 
3460 void TemplateTable::invokehandle(int byte_no) {
3461   transition(vtos, vtos);
3462   assert(byte_no == f1_byte, "use this argument");
3463 
3464   prepare_invoke(byte_no, rmethod, r0, r2);
3465   __ verify_method_ptr(r2);
3466   __ verify_oop(r2);
3467   __ null_check(r2);
3468 
3469   // FIXME: profile the LambdaForm also
3470 
3471   // r13 is safe to use here as a scratch reg because it is about to
3472   // be clobbered by jump_from_interpreted().
3473   __ profile_final_call(r13);
3474   __ profile_arguments_type(r13, rmethod, r4, true);
3475 
3476   __ jump_from_interpreted(rmethod, r0);
3477 }
3478 
3479 void TemplateTable::invokedynamic(int byte_no) {
3480   transition(vtos, vtos);
3481   assert(byte_no == f1_byte, "use this argument");
3482 
3483   prepare_invoke(byte_no, rmethod, r0);
3484 
3485   // r0: CallSite object (from cpool->resolved_references[])
3486   // rmethod: MH.linkToCallSite method (from f2)
3487 
3488   // Note:  r0_callsite is already pushed by prepare_invoke
3489 
3490   // %%% should make a type profile for any invokedynamic that takes a ref argument
3491   // profile this call
3492   __ profile_call(rbcp);
3493   __ profile_arguments_type(r3, rmethod, r13, false);
3494 
3495   __ verify_oop(r0);
3496 
3497   __ jump_from_interpreted(rmethod, r0);
3498 }
3499 
3500 
3501 //-----------------------------------------------------------------------------
3502 // Allocation
3503 
3504 void TemplateTable::_new() {
3505   transition(vtos, atos);
3506 
3507   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3508   Label slow_case;
3509   Label done;
3510   Label initialize_header;
3511   Label initialize_object; // including clearing the fields
3512 
3513   __ get_cpool_and_tags(r4, r0);
3514   // Make sure the class we're about to instantiate has been resolved.
3515   // This is done before loading InstanceKlass to be consistent with the order
3516   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3517   const int tags_offset = Array<u1>::base_offset_in_bytes();
3518   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3519   __ lea(rscratch1, Address(rscratch1, tags_offset));
3520   __ ldarb(rscratch1, rscratch1);
3521   __ cmp(rscratch1, JVM_CONSTANT_Class);
3522   __ br(Assembler::NE, slow_case);
3523 
3524   // get InstanceKlass
3525   __ load_resolved_klass_at_offset(r4, r3, r4, rscratch1);
3526 
3527   // make sure klass is initialized & doesn't have finalizer
3528   // make sure klass is fully initialized
3529   __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3530   __ cmp(rscratch1, InstanceKlass::fully_initialized);
3531   __ br(Assembler::NE, slow_case);
3532 
3533   // get instance_size in InstanceKlass (scaled to a count of bytes)
3534   __ ldrw(r3,
3535           Address(r4,
3536                   Klass::layout_helper_offset()));
3537   // test to see if it has a finalizer or is malformed in some way
3538   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3539 
3540   // Allocate the instance:
3541   //  If TLAB is enabled:
3542   //    Try to allocate in the TLAB.
3543   //    If fails, go to the slow path.
3544   //  Else If inline contiguous allocations are enabled:
3545   //    Try to allocate in eden.
3546   //    If fails due to heap end, go to slow path.
3547   //
3548   //  If TLAB is enabled OR inline contiguous is enabled:
3549   //    Initialize the allocation.
3550   //    Exit.
3551   //
3552   //  Go to slow path.
3553   const bool allow_shared_alloc =
3554     Universe::heap()->supports_inline_contig_alloc();
3555 
3556   if (UseTLAB) {
3557     __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case);
3558 
3559     if (ZeroTLAB) {
3560       // the fields have been already cleared
3561       __ b(initialize_header);
3562     } else {
3563       // initialize both the header and fields
3564       __ b(initialize_object);
3565     }
3566   } else {
3567     // Allocation in the shared Eden, if allowed.
3568     //
3569     // r3: instance size in bytes
3570     if (allow_shared_alloc) {
3571       __ eden_allocate(r0, r3, 0, r10, slow_case);
3572       __ incr_allocated_bytes(rthread, r3, 0, rscratch1);
3573     }
3574   }
3575 
3576   // If UseTLAB or allow_shared_alloc are true, the object is created above and
3577   // there is an initialize need. Otherwise, skip and go to the slow path.
3578   if (UseTLAB || allow_shared_alloc) {
3579     // The object is initialized before the header.  If the object size is
3580     // zero, go directly to the header initialization.
3581     __ bind(initialize_object);
3582     __ sub(r3, r3, sizeof(oopDesc));
3583     __ cbz(r3, initialize_header);
3584 
3585     // Initialize object fields
3586     {
3587       __ add(r2, r0, sizeof(oopDesc));
3588       Label loop;
3589       __ bind(loop);
3590       __ str(zr, Address(__ post(r2, BytesPerLong)));
3591       __ sub(r3, r3, BytesPerLong);
3592       __ cbnz(r3, loop);
3593     }
3594 
3595     // initialize object header only.
3596     __ bind(initialize_header);
3597     if (UseBiasedLocking) {
3598       __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3599     } else {
3600       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3601     }
3602     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3603     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3604     __ store_klass(r0, r4);      // store klass last
3605 
3606     {
3607       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3608       // Trigger dtrace event for fastpath
3609       __ push(atos); // save the return value
3610       __ call_VM_leaf(
3611            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3612       __ pop(atos); // restore the return value
3613 
3614     }
3615     __ b(done);
3616   }
3617 
3618   // slow case
3619   __ bind(slow_case);
3620   __ get_constant_pool(c_rarg1);
3621   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3622   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3623   __ verify_oop(r0);
3624 
3625   // continue
3626   __ bind(done);
3627   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3628   __ membar(Assembler::StoreStore);
3629 }
3630 
3631 void TemplateTable::newarray() {
3632   transition(itos, atos);
3633   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3634   __ mov(c_rarg2, r0);
3635   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3636           c_rarg1, c_rarg2);
3637   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3638   __ membar(Assembler::StoreStore);
3639 }
3640 
3641 void TemplateTable::anewarray() {
3642   transition(itos, atos);
3643   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3644   __ get_constant_pool(c_rarg1);
3645   __ mov(c_rarg3, r0);
3646   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3647           c_rarg1, c_rarg2, c_rarg3);
3648   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3649   __ membar(Assembler::StoreStore);
3650 }
3651 
3652 void TemplateTable::arraylength() {
3653   transition(atos, itos);
3654   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3655   __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3656 }
3657 
3658 void TemplateTable::checkcast()
3659 {
3660   transition(atos, atos);
3661   Label done, is_null, ok_is_subtype, quicked, resolved;
3662   __ cbz(r0, is_null);
3663 
3664   // Get cpool & tags index
3665   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3666   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3667   // See if bytecode has already been quicked
3668   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3669   __ lea(r1, Address(rscratch1, r19));
3670   __ ldarb(r1, r1);
3671   __ cmp(r1, JVM_CONSTANT_Class);
3672   __ br(Assembler::EQ, quicked);
3673 
3674   __ push(atos); // save receiver for result, and for GC
3675   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3676   // vm_result_2 has metadata result
3677   __ get_vm_result_2(r0, rthread);
3678   __ pop(r3); // restore receiver
3679   __ b(resolved);
3680 
3681   // Get superklass in r0 and subklass in r3
3682   __ bind(quicked);
3683   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3684   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1); // r0 = klass
3685 
3686   __ bind(resolved);
3687   __ load_klass(r19, r3);
3688 
3689   // Generate subtype check.  Blows r2, r5.  Object in r3.
3690   // Superklass in r0.  Subklass in r19.
3691   __ gen_subtype_check(r19, ok_is_subtype);
3692 
3693   // Come here on failure
3694   __ push(r3);
3695   // object is at TOS
3696   __ b(Interpreter::_throw_ClassCastException_entry);
3697 
3698   // Come here on success
3699   __ bind(ok_is_subtype);
3700   __ mov(r0, r3); // Restore object in r3
3701 
3702   // Collect counts on whether this test sees NULLs a lot or not.
3703   if (ProfileInterpreter) {
3704     __ b(done);
3705     __ bind(is_null);
3706     __ profile_null_seen(r2);
3707   } else {
3708     __ bind(is_null);   // same as 'done'
3709   }
3710   __ bind(done);
3711 }
3712 
3713 void TemplateTable::instanceof() {
3714   transition(atos, itos);
3715   Label done, is_null, ok_is_subtype, quicked, resolved;
3716   __ cbz(r0, is_null);
3717 
3718   // Get cpool & tags index
3719   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3720   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3721   // See if bytecode has already been quicked
3722   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3723   __ lea(r1, Address(rscratch1, r19));
3724   __ ldarb(r1, r1);
3725   __ cmp(r1, JVM_CONSTANT_Class);
3726   __ br(Assembler::EQ, quicked);
3727 
3728   __ push(atos); // save receiver for result, and for GC
3729   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3730   // vm_result_2 has metadata result
3731   __ get_vm_result_2(r0, rthread);
3732   __ pop(r3); // restore receiver
3733   __ verify_oop(r3);
3734   __ load_klass(r3, r3);
3735   __ b(resolved);
3736 
3737   // Get superklass in r0 and subklass in r3
3738   __ bind(quicked);
3739   __ load_klass(r3, r0);
3740   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1);
3741 
3742   __ bind(resolved);
3743 
3744   // Generate subtype check.  Blows r2, r5
3745   // Superklass in r0.  Subklass in r3.
3746   __ gen_subtype_check(r3, ok_is_subtype);
3747 
3748   // Come here on failure
3749   __ mov(r0, 0);
3750   __ b(done);
3751   // Come here on success
3752   __ bind(ok_is_subtype);
3753   __ mov(r0, 1);
3754 
3755   // Collect counts on whether this test sees NULLs a lot or not.
3756   if (ProfileInterpreter) {
3757     __ b(done);
3758     __ bind(is_null);
3759     __ profile_null_seen(r2);
3760   } else {
3761     __ bind(is_null);   // same as 'done'
3762   }
3763   __ bind(done);
3764   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
3765   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
3766 }
3767 
3768 //-----------------------------------------------------------------------------
3769 // Breakpoints
3770 void TemplateTable::_breakpoint() {
3771   // Note: We get here even if we are single stepping..
3772   // jbug inists on setting breakpoints at every bytecode
3773   // even if we are in single step mode.
3774 
3775   transition(vtos, vtos);
3776 
3777   // get the unpatched byte code
3778   __ get_method(c_rarg1);
3779   __ call_VM(noreg,
3780              CAST_FROM_FN_PTR(address,
3781                               InterpreterRuntime::get_original_bytecode_at),
3782              c_rarg1, rbcp);
3783   __ mov(r19, r0);
3784 
3785   // post the breakpoint event
3786   __ call_VM(noreg,
3787              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3788              rmethod, rbcp);
3789 
3790   // complete the execution of original bytecode
3791   __ mov(rscratch1, r19);
3792   __ dispatch_only_normal(vtos);
3793 }
3794 
3795 //-----------------------------------------------------------------------------
3796 // Exceptions
3797 
3798 void TemplateTable::athrow() {
3799   transition(atos, vtos);
3800   __ null_check(r0);
3801   __ b(Interpreter::throw_exception_entry());
3802 }
3803 
3804 //-----------------------------------------------------------------------------
3805 // Synchronization
3806 //
3807 // Note: monitorenter & exit are symmetric routines; which is reflected
3808 //       in the assembly code structure as well
3809 //
3810 // Stack layout:
3811 //
3812 // [expressions  ] <--- esp               = expression stack top
3813 // ..
3814 // [expressions  ]
3815 // [monitor entry] <--- monitor block top = expression stack bot
3816 // ..
3817 // [monitor entry]
3818 // [frame data   ] <--- monitor block bot
3819 // ...
3820 // [saved rbp    ] <--- rbp
3821 void TemplateTable::monitorenter()
3822 {
3823   transition(atos, vtos);
3824 
3825   // check for NULL object
3826   __ null_check(r0);
3827 
3828   const Address monitor_block_top(
3829         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3830   const Address monitor_block_bot(
3831         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3832   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3833 
3834   Label allocated;
3835 
3836   // initialize entry pointer
3837   __ mov(c_rarg1, zr); // points to free slot or NULL
3838 
3839   // find a free slot in the monitor block (result in c_rarg1)
3840   {
3841     Label entry, loop, exit;
3842     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
3843                                         // starting with top-most entry
3844     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3845 
3846     __ b(entry);
3847 
3848     __ bind(loop);
3849     // check if current entry is used
3850     // if not used then remember entry in c_rarg1
3851     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
3852     __ cmp(zr, rscratch1);
3853     __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
3854     // check if current entry is for same object
3855     __ cmp(r0, rscratch1);
3856     // if same object then stop searching
3857     __ br(Assembler::EQ, exit);
3858     // otherwise advance to next entry
3859     __ add(c_rarg3, c_rarg3, entry_size);
3860     __ bind(entry);
3861     // check if bottom reached
3862     __ cmp(c_rarg3, c_rarg2);
3863     // if not at bottom then check this entry
3864     __ br(Assembler::NE, loop);
3865     __ bind(exit);
3866   }
3867 
3868   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
3869                             // if found, continue with that on
3870 
3871   // allocate one if there's no free slot
3872   {
3873     Label entry, loop;
3874     // 1. compute new pointers            // rsp: old expression stack top
3875     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
3876     __ sub(esp, esp, entry_size);         // move expression stack top
3877     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3878     __ mov(c_rarg3, esp);                 // set start value for copy loop
3879     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
3880 
3881     __ sub(sp, sp, entry_size);           // make room for the monitor
3882 
3883     __ b(entry);
3884     // 2. move expression stack contents
3885     __ bind(loop);
3886     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3887                                                    // word from old location
3888     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
3889     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
3890     __ bind(entry);
3891     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
3892     __ br(Assembler::NE, loop);      // if not at bottom then
3893                                      // copy next word
3894   }
3895 
3896   // call run-time routine
3897   // c_rarg1: points to monitor entry
3898   __ bind(allocated);
3899 
3900   // Increment bcp to point to the next bytecode, so exception
3901   // handling for async. exceptions work correctly.
3902   // The object has already been poped from the stack, so the
3903   // expression stack looks correct.
3904   __ increment(rbcp);
3905 
3906   // store object
3907   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3908   __ lock_object(c_rarg1);
3909 
3910   // check to make sure this monitor doesn't cause stack overflow after locking
3911   __ save_bcp();  // in case of exception
3912   __ generate_stack_overflow_check(0);
3913 
3914   // The bcp has already been incremented. Just need to dispatch to
3915   // next instruction.
3916   __ dispatch_next(vtos);
3917 }
3918 
3919 
3920 void TemplateTable::monitorexit()
3921 {
3922   transition(atos, vtos);
3923 
3924   // check for NULL object
3925   __ null_check(r0);
3926 
3927   const Address monitor_block_top(
3928         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3929   const Address monitor_block_bot(
3930         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3931   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3932 
3933   Label found;
3934 
3935   // find matching slot
3936   {
3937     Label entry, loop;
3938     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
3939                                         // starting with top-most entry
3940     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3941                                         // of monitor block
3942     __ b(entry);
3943 
3944     __ bind(loop);
3945     // check if current entry is for same object
3946     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3947     __ cmp(r0, rscratch1);
3948     // if same object then stop searching
3949     __ br(Assembler::EQ, found);
3950     // otherwise advance to next entry
3951     __ add(c_rarg1, c_rarg1, entry_size);
3952     __ bind(entry);
3953     // check if bottom reached
3954     __ cmp(c_rarg1, c_rarg2);
3955     // if not at bottom then check this entry
3956     __ br(Assembler::NE, loop);
3957   }
3958 
3959   // error handling. Unlocking was not block-structured
3960   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3961                    InterpreterRuntime::throw_illegal_monitor_state_exception));
3962   __ should_not_reach_here();
3963 
3964   // call run-time routine
3965   __ bind(found);
3966   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
3967   __ unlock_object(c_rarg1);
3968   __ pop_ptr(r0); // discard object
3969 }
3970 
3971 
3972 // Wide instructions
3973 void TemplateTable::wide()
3974 {
3975   __ load_unsigned_byte(r19, at_bcp(1));
3976   __ mov(rscratch1, (address)Interpreter::_wentry_point);
3977   __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
3978   __ br(rscratch1);
3979 }
3980 
3981 
3982 // Multi arrays
3983 void TemplateTable::multianewarray() {
3984   transition(vtos, atos);
3985   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
3986   // last dim is on top of stack; we want address of first one:
3987   // first_addr = last_addr + (ndims - 1) * wordSize
3988   __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
3989   __ sub(c_rarg1, c_rarg1, wordSize);
3990   call_VM(r0,
3991           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3992           c_rarg1);
3993   __ load_unsigned_byte(r1, at_bcp(3));
3994   __ lea(esp, Address(esp, r1, Address::uxtw(3)));
3995 }