1 /*
   2  * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/barrierSetAssembler.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/interp_masm.hpp"
  32 #include "interpreter/templateTable.hpp"
  33 #include "memory/universe.hpp"
  34 #include "oops/methodData.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/objArrayKlass.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/frame.inline.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "runtime/stubRoutines.hpp"
  42 #include "runtime/synchronizer.hpp"
  43 
  44 #define __ _masm->
  45 
  46 // Platform-dependent initialization
  47 
  48 void TemplateTable::pd_initialize() {
  49   // No aarch64 specific initialization
  50 }
  51 
  52 // Address computation: local variables
  53 
  54 static inline Address iaddress(int n) {
  55   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  56 }
  57 
  58 static inline Address laddress(int n) {
  59   return iaddress(n + 1);
  60 }
  61 
  62 static inline Address faddress(int n) {
  63   return iaddress(n);
  64 }
  65 
  66 static inline Address daddress(int n) {
  67   return laddress(n);
  68 }
  69 
  70 static inline Address aaddress(int n) {
  71   return iaddress(n);
  72 }
  73 
  74 static inline Address iaddress(Register r) {
  75   return Address(rlocals, r, Address::lsl(3));
  76 }
  77 
  78 static inline Address laddress(Register r, Register scratch,
  79                                InterpreterMacroAssembler* _masm) {
  80   __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
  81   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  82 }
  83 
  84 static inline Address faddress(Register r) {
  85   return iaddress(r);
  86 }
  87 
  88 static inline Address daddress(Register r, Register scratch,
  89                                InterpreterMacroAssembler* _masm) {
  90   return laddress(r, scratch, _masm);
  91 }
  92 
  93 static inline Address aaddress(Register r) {
  94   return iaddress(r);
  95 }
  96 
  97 static inline Address at_rsp() {
  98   return Address(esp, 0);
  99 }
 100 
 101 // At top of Java expression stack which may be different than esp().  It
 102 // isn't for category 1 objects.
 103 static inline Address at_tos   () {
 104   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 105 }
 106 
 107 static inline Address at_tos_p1() {
 108   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 109 }
 110 
 111 static inline Address at_tos_p2() {
 112   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 113 }
 114 
 115 static inline Address at_tos_p3() {
 116   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 117 }
 118 
 119 static inline Address at_tos_p4() {
 120   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 121 }
 122 
 123 static inline Address at_tos_p5() {
 124   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 125 }
 126 
 127 // Condition conversion
 128 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 129   switch (cc) {
 130   case TemplateTable::equal        : return Assembler::NE;
 131   case TemplateTable::not_equal    : return Assembler::EQ;
 132   case TemplateTable::less         : return Assembler::GE;
 133   case TemplateTable::less_equal   : return Assembler::GT;
 134   case TemplateTable::greater      : return Assembler::LE;
 135   case TemplateTable::greater_equal: return Assembler::LT;
 136   }
 137   ShouldNotReachHere();
 138   return Assembler::EQ;
 139 }
 140 
 141 
 142 // Miscelaneous helper routines
 143 // Store an oop (or NULL) at the Address described by obj.
 144 // If val == noreg this means store a NULL
 145 static void do_oop_store(InterpreterMacroAssembler* _masm,
 146                          Address dst,
 147                          Register val,
 148                          DecoratorSet decorators) {
 149   assert(val == noreg || val == r0, "parameter is just for looks");
 150   __ store_heap_oop(dst, val, r10, r1, noreg, decorators); 
 151 }
 152 
 153 static void do_oop_load(InterpreterMacroAssembler* _masm,
 154                         Address src,
 155                         Register dst,
 156                         DecoratorSet decorators) {
 157   __ load_heap_oop(dst, src, r10, r1, decorators);
 158 }
 159 
 160 Address TemplateTable::at_bcp(int offset) {
 161   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 162   return Address(rbcp, offset);
 163 }
 164 
 165 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 166                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 167                                    int byte_no)
 168 {
 169   if (!RewriteBytecodes)  return;
 170   Label L_patch_done;
 171 
 172   switch (bc) {
 173   case Bytecodes::_fast_qputfield:
 174   case Bytecodes::_fast_aputfield:
 175   case Bytecodes::_fast_bputfield:
 176   case Bytecodes::_fast_zputfield:
 177   case Bytecodes::_fast_cputfield:
 178   case Bytecodes::_fast_dputfield:
 179   case Bytecodes::_fast_fputfield:
 180   case Bytecodes::_fast_iputfield:
 181   case Bytecodes::_fast_lputfield:
 182   case Bytecodes::_fast_sputfield:
 183     {
 184       // We skip bytecode quickening for putfield instructions when
 185       // the put_code written to the constant pool cache is zero.
 186       // This is required so that every execution of this instruction
 187       // calls out to InterpreterRuntime::resolve_get_put to do
 188       // additional, required work.
 189       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 190       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 191       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 192       __ movw(bc_reg, bc);
 193       __ cbzw(temp_reg, L_patch_done);  // don't patch
 194     }
 195     break;
 196   default:
 197     assert(byte_no == -1, "sanity");
 198     // the pair bytecodes have already done the load.
 199     if (load_bc_into_bc_reg) {
 200       __ movw(bc_reg, bc);
 201     }
 202   }
 203 
 204   if (JvmtiExport::can_post_breakpoint()) {
 205     Label L_fast_patch;
 206     // if a breakpoint is present we can't rewrite the stream directly
 207     __ load_unsigned_byte(temp_reg, at_bcp(0));
 208     __ cmpw(temp_reg, Bytecodes::_breakpoint);
 209     __ br(Assembler::NE, L_fast_patch);
 210     // Let breakpoint table handling rewrite to quicker bytecode
 211     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 212     __ b(L_patch_done);
 213     __ bind(L_fast_patch);
 214   }
 215 
 216 #ifdef ASSERT
 217   Label L_okay;
 218   __ load_unsigned_byte(temp_reg, at_bcp(0));
 219   __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
 220   __ br(Assembler::EQ, L_okay);
 221   __ cmpw(temp_reg, bc_reg);
 222   __ br(Assembler::EQ, L_okay);
 223   __ stop("patching the wrong bytecode");
 224   __ bind(L_okay);
 225 #endif
 226 
 227   // patch bytecode
 228   __ strb(bc_reg, at_bcp(0));
 229   __ bind(L_patch_done);
 230 }
 231 
 232 
 233 // Individual instructions
 234 
 235 void TemplateTable::nop() {
 236   transition(vtos, vtos);
 237   // nothing to do
 238 }
 239 
 240 void TemplateTable::shouldnotreachhere() {
 241   transition(vtos, vtos);
 242   __ stop("shouldnotreachhere bytecode");
 243 }
 244 
 245 void TemplateTable::aconst_null()
 246 {
 247   transition(vtos, atos);
 248   __ mov(r0, 0);
 249 }
 250 
 251 void TemplateTable::iconst(int value)
 252 {
 253   transition(vtos, itos);
 254   __ mov(r0, value);
 255 }
 256 
 257 void TemplateTable::lconst(int value)
 258 {
 259   __ mov(r0, value);
 260 }
 261 
 262 void TemplateTable::fconst(int value)
 263 {
 264   transition(vtos, ftos);
 265   switch (value) {
 266   case 0:
 267     __ fmovs(v0, zr);
 268     break;
 269   case 1:
 270     __ fmovs(v0, 1.0);
 271     break;
 272   case 2:
 273     __ fmovs(v0, 2.0);
 274     break;
 275   default:
 276     ShouldNotReachHere();
 277     break;
 278   }
 279 }
 280 
 281 void TemplateTable::dconst(int value)
 282 {
 283   transition(vtos, dtos);
 284   switch (value) {
 285   case 0:
 286     __ fmovd(v0, zr);
 287     break;
 288   case 1:
 289     __ fmovd(v0, 1.0);
 290     break;
 291   case 2:
 292     __ fmovd(v0, 2.0);
 293     break;
 294   default:
 295     ShouldNotReachHere();
 296     break;
 297   }
 298 }
 299 
 300 void TemplateTable::bipush()
 301 {
 302   transition(vtos, itos);
 303   __ load_signed_byte32(r0, at_bcp(1));
 304 }
 305 
 306 void TemplateTable::sipush()
 307 {
 308   transition(vtos, itos);
 309   __ load_unsigned_short(r0, at_bcp(1));
 310   __ revw(r0, r0);
 311   __ asrw(r0, r0, 16);
 312 }
 313 
 314 void TemplateTable::ldc(bool wide)
 315 {
 316   transition(vtos, vtos);
 317   Label call_ldc, notFloat, notClass, notInt, Done;
 318 
 319   if (wide) {
 320     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 321   } else {
 322     __ load_unsigned_byte(r1, at_bcp(1));
 323   }
 324   __ get_cpool_and_tags(r2, r0);
 325 
 326   const int base_offset = ConstantPool::header_size() * wordSize;
 327   const int tags_offset = Array<u1>::base_offset_in_bytes();
 328 
 329   // get type
 330   __ add(r3, r1, tags_offset);
 331   __ lea(r3, Address(r0, r3));
 332   __ ldarb(r3, r3);
 333 
 334   // unresolved class - get the resolved class
 335   __ cmp(r3, (u1)JVM_CONSTANT_UnresolvedClass);
 336   __ br(Assembler::EQ, call_ldc);
 337 
 338   // unresolved class in error state - call into runtime to throw the error
 339   // from the first resolution attempt
 340   __ cmp(r3, (u1)JVM_CONSTANT_UnresolvedClassInError);
 341   __ br(Assembler::EQ, call_ldc);
 342 
 343   // resolved class - need to call vm to get java mirror of the class
 344   __ cmp(r3, (u1)JVM_CONSTANT_Class);
 345   __ br(Assembler::NE, notClass);
 346 
 347   __ bind(call_ldc);
 348   __ mov(c_rarg1, wide);
 349   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 350   __ push_ptr(r0);
 351   __ verify_oop(r0);
 352   __ b(Done);
 353 
 354   __ bind(notClass);
 355   __ cmp(r3, (u1)JVM_CONSTANT_Float);
 356   __ br(Assembler::NE, notFloat);
 357   // ftos
 358   __ adds(r1, r2, r1, Assembler::LSL, 3);
 359   __ ldrs(v0, Address(r1, base_offset));
 360   __ push_f();
 361   __ b(Done);
 362 
 363   __ bind(notFloat);
 364 
 365   __ cmp(r3, (u1)JVM_CONSTANT_Integer);
 366   __ br(Assembler::NE, notInt);
 367 
 368   // itos
 369   __ adds(r1, r2, r1, Assembler::LSL, 3);
 370   __ ldrw(r0, Address(r1, base_offset));
 371   __ push_i(r0);
 372   __ b(Done);
 373 
 374   __ bind(notInt);
 375   condy_helper(Done);
 376 
 377   __ bind(Done);
 378 }
 379 
 380 // Fast path for caching oop constants.
 381 void TemplateTable::fast_aldc(bool wide)
 382 {
 383   transition(vtos, atos);
 384 
 385   Register result = r0;
 386   Register tmp = r1;
 387   Register rarg = r2;
 388 
 389   int index_size = wide ? sizeof(u2) : sizeof(u1);
 390 
 391   Label resolved;
 392 
 393   // We are resolved if the resolved reference cache entry contains a
 394   // non-null object (String, MethodType, etc.)
 395   assert_different_registers(result, tmp);
 396   __ get_cache_index_at_bcp(tmp, 1, index_size);
 397   __ load_resolved_reference_at_index(result, tmp);
 398   __ cbnz(result, resolved);
 399 
 400   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 401 
 402   // first time invocation - must resolve first
 403   __ mov(rarg, (int)bytecode());
 404   __ call_VM(result, entry, rarg);
 405 
 406   __ bind(resolved);
 407 
 408   { // Check for the null sentinel.
 409     // If we just called the VM, it already did the mapping for us,
 410     // but it's harmless to retry.
 411     Label notNull;
 412 
 413     // Stash null_sentinel address to get its value later
 414     __ movptr(rarg, (uintptr_t)Universe::the_null_sentinel_addr());
 415     __ ldr(tmp, Address(rarg));
 416     __ cmpoop(result, tmp);
 417     __ br(Assembler::NE, notNull);
 418     __ mov(result, 0);  // NULL object reference
 419     __ bind(notNull);
 420   }
 421 
 422   if (VerifyOops) {
 423     // Safe to call with 0 result
 424     __ verify_oop(result);
 425   }
 426 }
 427 
 428 void TemplateTable::ldc2_w()
 429 {
 430   transition(vtos, vtos);
 431   Label notDouble, notLong, Done;
 432   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 433 
 434   __ get_cpool_and_tags(r1, r2);
 435   const int base_offset = ConstantPool::header_size() * wordSize;
 436   const int tags_offset = Array<u1>::base_offset_in_bytes();
 437 
 438   // get type
 439   __ lea(r2, Address(r2, r0, Address::lsl(0)));
 440   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 441   __ cmpw(r2, (int)JVM_CONSTANT_Double);
 442   __ br(Assembler::NE, notDouble);
 443 
 444   // dtos
 445   __ lea (r2, Address(r1, r0, Address::lsl(3)));
 446   __ ldrd(v0, Address(r2, base_offset));
 447   __ push_d();
 448   __ b(Done);
 449 
 450   __ bind(notDouble);
 451   __ cmpw(r2, (int)JVM_CONSTANT_Long);
 452   __ br(Assembler::NE, notLong);
 453 
 454   // ltos
 455   __ lea(r0, Address(r1, r0, Address::lsl(3)));
 456   __ ldr(r0, Address(r0, base_offset));
 457   __ push_l();
 458   __ b(Done);
 459 
 460   __ bind(notLong);
 461   condy_helper(Done);
 462 
 463   __ bind(Done);
 464 }
 465 
 466 void TemplateTable::condy_helper(Label& Done)
 467 {
 468   Register obj = r0;
 469   Register rarg = r1;
 470   Register flags = r2;
 471   Register off = r3;
 472 
 473   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 474 
 475   __ mov(rarg, (int) bytecode());
 476   __ call_VM(obj, entry, rarg);
 477 
 478   __ get_vm_result_2(flags, rthread);
 479 
 480   // VMr = obj = base address to find primitive value to push
 481   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 482   __ mov(off, flags);
 483   __ andw(off, off, ConstantPoolCacheEntry::field_index_mask);
 484 
 485   const Address field(obj, off);
 486 
 487   // What sort of thing are we loading?
 488   // x86 uses a shift and mask or wings it with a shift plus assert
 489   // the mask is not needed. aarch64 just uses bitfield extract
 490   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,
 491            ConstantPoolCacheEntry::tos_state_bits);
 492 
 493   switch (bytecode()) {
 494     case Bytecodes::_ldc:
 495     case Bytecodes::_ldc_w:
 496       {
 497         // tos in (itos, ftos, stos, btos, ctos, ztos)
 498         Label notInt, notFloat, notShort, notByte, notChar, notBool;
 499         __ cmpw(flags, itos);
 500         __ br(Assembler::NE, notInt);
 501         // itos
 502         __ ldrw(r0, field);
 503         __ push(itos);
 504         __ b(Done);
 505 
 506         __ bind(notInt);
 507         __ cmpw(flags, ftos);
 508         __ br(Assembler::NE, notFloat);
 509         // ftos
 510         __ load_float(field);
 511         __ push(ftos);
 512         __ b(Done);
 513 
 514         __ bind(notFloat);
 515         __ cmpw(flags, stos);
 516         __ br(Assembler::NE, notShort);
 517         // stos
 518         __ load_signed_short(r0, field);
 519         __ push(stos);
 520         __ b(Done);
 521 
 522         __ bind(notShort);
 523         __ cmpw(flags, btos);
 524         __ br(Assembler::NE, notByte);
 525         // btos
 526         __ load_signed_byte(r0, field);
 527         __ push(btos);
 528         __ b(Done);
 529 
 530         __ bind(notByte);
 531         __ cmpw(flags, ctos);
 532         __ br(Assembler::NE, notChar);
 533         // ctos
 534         __ load_unsigned_short(r0, field);
 535         __ push(ctos);
 536         __ b(Done);
 537 
 538         __ bind(notChar);
 539         __ cmpw(flags, ztos);
 540         __ br(Assembler::NE, notBool);
 541         // ztos
 542         __ load_signed_byte(r0, field);
 543         __ push(ztos);
 544         __ b(Done);
 545 
 546         __ bind(notBool);
 547         break;
 548       }
 549 
 550     case Bytecodes::_ldc2_w:
 551       {
 552         Label notLong, notDouble;
 553         __ cmpw(flags, ltos);
 554         __ br(Assembler::NE, notLong);
 555         // ltos
 556         __ ldr(r0, field);
 557         __ push(ltos);
 558         __ b(Done);
 559 
 560         __ bind(notLong);
 561         __ cmpw(flags, dtos);
 562         __ br(Assembler::NE, notDouble);
 563         // dtos
 564         __ load_double(field);
 565         __ push(dtos);
 566         __ b(Done);
 567 
 568        __ bind(notDouble);
 569         break;
 570       }
 571 
 572     default:
 573       ShouldNotReachHere();
 574     }
 575 
 576     __ stop("bad ldc/condy");
 577 }
 578 
 579 void TemplateTable::locals_index(Register reg, int offset)
 580 {
 581   __ ldrb(reg, at_bcp(offset));
 582   __ neg(reg, reg);
 583 }
 584 
 585 void TemplateTable::iload() {
 586   iload_internal();
 587 }
 588 
 589 void TemplateTable::nofast_iload() {
 590   iload_internal(may_not_rewrite);
 591 }
 592 
 593 void TemplateTable::iload_internal(RewriteControl rc) {
 594   transition(vtos, itos);
 595   if (RewriteFrequentPairs && rc == may_rewrite) {
 596     Label rewrite, done;
 597     Register bc = r4;
 598 
 599     // get next bytecode
 600     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 601 
 602     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 603     // last two iloads in a pair.  Comparing against fast_iload means that
 604     // the next bytecode is neither an iload or a caload, and therefore
 605     // an iload pair.
 606     __ cmpw(r1, Bytecodes::_iload);
 607     __ br(Assembler::EQ, done);
 608 
 609     // if _fast_iload rewrite to _fast_iload2
 610     __ cmpw(r1, Bytecodes::_fast_iload);
 611     __ movw(bc, Bytecodes::_fast_iload2);
 612     __ br(Assembler::EQ, rewrite);
 613 
 614     // if _caload rewrite to _fast_icaload
 615     __ cmpw(r1, Bytecodes::_caload);
 616     __ movw(bc, Bytecodes::_fast_icaload);
 617     __ br(Assembler::EQ, rewrite);
 618 
 619     // else rewrite to _fast_iload
 620     __ movw(bc, Bytecodes::_fast_iload);
 621 
 622     // rewrite
 623     // bc: new bytecode
 624     __ bind(rewrite);
 625     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 626     __ bind(done);
 627 
 628   }
 629 
 630   // do iload, get the local value into tos
 631   locals_index(r1);
 632   __ ldr(r0, iaddress(r1));
 633 
 634 }
 635 
 636 void TemplateTable::fast_iload2()
 637 {
 638   transition(vtos, itos);
 639   locals_index(r1);
 640   __ ldr(r0, iaddress(r1));
 641   __ push(itos);
 642   locals_index(r1, 3);
 643   __ ldr(r0, iaddress(r1));
 644 }
 645 
 646 void TemplateTable::fast_iload()
 647 {
 648   transition(vtos, itos);
 649   locals_index(r1);
 650   __ ldr(r0, iaddress(r1));
 651 }
 652 
 653 void TemplateTable::lload()
 654 {
 655   transition(vtos, ltos);
 656   __ ldrb(r1, at_bcp(1));
 657   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 658   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 659 }
 660 
 661 void TemplateTable::fload()
 662 {
 663   transition(vtos, ftos);
 664   locals_index(r1);
 665   // n.b. we use ldrd here because this is a 64 bit slot
 666   // this is comparable to the iload case
 667   __ ldrd(v0, faddress(r1));
 668 }
 669 
 670 void TemplateTable::dload()
 671 {
 672   transition(vtos, dtos);
 673   __ ldrb(r1, at_bcp(1));
 674   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 675   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 676 }
 677 
 678 void TemplateTable::aload()
 679 {
 680   transition(vtos, atos);
 681   locals_index(r1);
 682   __ ldr(r0, iaddress(r1));
 683 }
 684 
 685 void TemplateTable::locals_index_wide(Register reg) {
 686   __ ldrh(reg, at_bcp(2));
 687   __ rev16w(reg, reg);
 688   __ neg(reg, reg);
 689 }
 690 
 691 void TemplateTable::wide_iload() {
 692   transition(vtos, itos);
 693   locals_index_wide(r1);
 694   __ ldr(r0, iaddress(r1));
 695 }
 696 
 697 void TemplateTable::wide_lload()
 698 {
 699   transition(vtos, ltos);
 700   __ ldrh(r1, at_bcp(2));
 701   __ rev16w(r1, r1);
 702   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 703   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 704 }
 705 
 706 void TemplateTable::wide_fload()
 707 {
 708   transition(vtos, ftos);
 709   locals_index_wide(r1);
 710   // n.b. we use ldrd here because this is a 64 bit slot
 711   // this is comparable to the iload case
 712   __ ldrd(v0, faddress(r1));
 713 }
 714 
 715 void TemplateTable::wide_dload()
 716 {
 717   transition(vtos, dtos);
 718   __ ldrh(r1, at_bcp(2));
 719   __ rev16w(r1, r1);
 720   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 721   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 722 }
 723 
 724 void TemplateTable::wide_aload()
 725 {
 726   transition(vtos, atos);
 727   locals_index_wide(r1);
 728   __ ldr(r0, aaddress(r1));
 729 }
 730 
 731 void TemplateTable::index_check(Register array, Register index)
 732 {
 733   // destroys r1, rscratch1
 734   // check array
 735   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 736   // sign extend index for use by indexed load
 737   // __ movl2ptr(index, index);
 738   // check index
 739   Register length = rscratch1;
 740   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 741   __ cmpw(index, length);
 742   if (index != r1) {
 743     // ??? convention: move aberrant index into r1 for exception message
 744     assert(r1 != array, "different registers");
 745     __ mov(r1, index);
 746   }
 747   Label ok;
 748   __ br(Assembler::LO, ok);
 749   // ??? convention: move array into r3 for exception message
 750    __ mov(r3, array);
 751    __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 752    __ br(rscratch1);
 753   __ bind(ok);
 754 }
 755 
 756 void TemplateTable::iaload()
 757 {
 758   transition(itos, itos);
 759   __ mov(r1, r0);
 760   __ pop_ptr(r0);
 761   // r0: array
 762   // r1: index
 763   index_check(r0, r1); // leaves index in r1, kills rscratch1
 764   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
 765   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(2)), noreg, noreg);
 766 }
 767 
 768 void TemplateTable::laload()
 769 {
 770   transition(itos, ltos);
 771   __ mov(r1, r0);
 772   __ pop_ptr(r0);
 773   // r0: array
 774   // r1: index
 775   index_check(r0, r1); // leaves index in r1, kills rscratch1
 776   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
 777   __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(3)), noreg, noreg);
 778 }
 779 
 780 void TemplateTable::faload()
 781 {
 782   transition(itos, ftos);
 783   __ mov(r1, r0);
 784   __ pop_ptr(r0);
 785   // r0: array
 786   // r1: index
 787   index_check(r0, r1); // leaves index in r1, kills rscratch1
 788   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
 789   __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(2)), noreg, noreg);
 790 }
 791 
 792 void TemplateTable::daload()
 793 {
 794   transition(itos, dtos);
 795   __ mov(r1, r0);
 796   __ pop_ptr(r0);
 797   // r0: array
 798   // r1: index
 799   index_check(r0, r1); // leaves index in r1, kills rscratch1
 800   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
 801   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(3)), noreg, noreg);
 802 }
 803 
 804 void TemplateTable::aaload()
 805 {
 806   transition(itos, atos);
 807   __ mov(r1, r0);
 808   __ pop_ptr(r0);
 809   // r0: array
 810   // r1: index
 811   index_check(r0, r1); // leaves index in r1, kills rscratch1
 812   if (ValueArrayFlatten) {
 813     Label is_flat_array, done;
 814 
 815     __ test_flattened_array_oop(r0, r8 /*temp*/, is_flat_array); 
 816     __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 817     do_oop_load(_masm, Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)), r0, IS_ARRAY);
 818 
 819     __ b(done);
 820     __ bind(is_flat_array);
 821     __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), r0, r1);
 822     __ bind(done);
 823   } else {
 824     __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 825     do_oop_load(_masm, Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)), r0, IS_ARRAY);
 826   }
 827 }
 828 
 829 void TemplateTable::baload()
 830 {
 831   transition(itos, itos);
 832   __ mov(r1, r0);
 833   __ pop_ptr(r0);
 834   // r0: array
 835   // r1: index
 836   index_check(r0, r1); // leaves index in r1, kills rscratch1
 837   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
 838   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(0)), noreg, noreg);
 839 }
 840 
 841 void TemplateTable::caload()
 842 {
 843   transition(itos, itos);
 844   __ mov(r1, r0);
 845   __ pop_ptr(r0);
 846   // r0: array
 847   // r1: index
 848   index_check(r0, r1); // leaves index in r1, kills rscratch1
 849   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 850   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 851 }
 852 
 853 // iload followed by caload frequent pair
 854 void TemplateTable::fast_icaload()
 855 {
 856   transition(vtos, itos);
 857   // load index out of locals
 858   locals_index(r2);
 859   __ ldr(r1, iaddress(r2));
 860 
 861   __ pop_ptr(r0);
 862 
 863   // r0: array
 864   // r1: index
 865   index_check(r0, r1); // leaves index in r1, kills rscratch1
 866   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 867   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 868 }
 869 
 870 void TemplateTable::saload()
 871 {
 872   transition(itos, itos);
 873   __ mov(r1, r0);
 874   __ pop_ptr(r0);
 875   // r0: array
 876   // r1: index
 877   index_check(r0, r1); // leaves index in r1, kills rscratch1
 878   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_SHORT) >> 1);
 879   __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 880 }
 881 
 882 void TemplateTable::iload(int n)
 883 {
 884   transition(vtos, itos);
 885   __ ldr(r0, iaddress(n));
 886 }
 887 
 888 void TemplateTable::lload(int n)
 889 {
 890   transition(vtos, ltos);
 891   __ ldr(r0, laddress(n));
 892 }
 893 
 894 void TemplateTable::fload(int n)
 895 {
 896   transition(vtos, ftos);
 897   __ ldrs(v0, faddress(n));
 898 }
 899 
 900 void TemplateTable::dload(int n)
 901 {
 902   transition(vtos, dtos);
 903   __ ldrd(v0, daddress(n));
 904 }
 905 
 906 void TemplateTable::aload(int n)
 907 {
 908   transition(vtos, atos);
 909   __ ldr(r0, iaddress(n));
 910 }
 911 
 912 void TemplateTable::aload_0() {
 913   aload_0_internal();
 914 }
 915 
 916 void TemplateTable::nofast_aload_0() {
 917   aload_0_internal(may_not_rewrite);
 918 }
 919 
 920 void TemplateTable::aload_0_internal(RewriteControl rc) {
 921   // According to bytecode histograms, the pairs:
 922   //
 923   // _aload_0, _fast_igetfield
 924   // _aload_0, _fast_agetfield
 925   // _aload_0, _fast_fgetfield
 926   //
 927   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 928   // _aload_0 bytecode checks if the next bytecode is either
 929   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 930   // rewrites the current bytecode into a pair bytecode; otherwise it
 931   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 932   // the pair check anymore.
 933   //
 934   // Note: If the next bytecode is _getfield, the rewrite must be
 935   //       delayed, otherwise we may miss an opportunity for a pair.
 936   //
 937   // Also rewrite frequent pairs
 938   //   aload_0, aload_1
 939   //   aload_0, iload_1
 940   // These bytecodes with a small amount of code are most profitable
 941   // to rewrite
 942   if (RewriteFrequentPairs && rc == may_rewrite) {
 943     Label rewrite, done;
 944     const Register bc = r4;
 945 
 946     // get next bytecode
 947     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 948 
 949     // if _getfield then wait with rewrite
 950     __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
 951     __ br(Assembler::EQ, done);
 952 
 953     // if _igetfield then rewrite to _fast_iaccess_0
 954     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 955     __ cmpw(r1, Bytecodes::_fast_igetfield);
 956     __ movw(bc, Bytecodes::_fast_iaccess_0);
 957     __ br(Assembler::EQ, rewrite);
 958 
 959     // if _agetfield then rewrite to _fast_aaccess_0
 960     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 961     __ cmpw(r1, Bytecodes::_fast_agetfield);
 962     __ movw(bc, Bytecodes::_fast_aaccess_0);
 963     __ br(Assembler::EQ, rewrite);
 964 
 965     // if _fgetfield then rewrite to _fast_faccess_0
 966     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 967     __ cmpw(r1, Bytecodes::_fast_fgetfield);
 968     __ movw(bc, Bytecodes::_fast_faccess_0);
 969     __ br(Assembler::EQ, rewrite);
 970 
 971     // else rewrite to _fast_aload0
 972     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 973     __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
 974 
 975     // rewrite
 976     // bc: new bytecode
 977     __ bind(rewrite);
 978     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 979 
 980     __ bind(done);
 981   }
 982 
 983   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 984   aload(0);
 985 }
 986 
 987 void TemplateTable::istore()
 988 {
 989   transition(itos, vtos);
 990   locals_index(r1);
 991   // FIXME: We're being very pernickerty here storing a jint in a
 992   // local with strw, which costs an extra instruction over what we'd
 993   // be able to do with a simple str.  We should just store the whole
 994   // word.
 995   __ lea(rscratch1, iaddress(r1));
 996   __ strw(r0, Address(rscratch1));
 997 }
 998 
 999 void TemplateTable::lstore()
1000 {
1001   transition(ltos, vtos);
1002   locals_index(r1);
1003   __ str(r0, laddress(r1, rscratch1, _masm));
1004 }
1005 
1006 void TemplateTable::fstore() {
1007   transition(ftos, vtos);
1008   locals_index(r1);
1009   __ lea(rscratch1, iaddress(r1));
1010   __ strs(v0, Address(rscratch1));
1011 }
1012 
1013 void TemplateTable::dstore() {
1014   transition(dtos, vtos);
1015   locals_index(r1);
1016   __ strd(v0, daddress(r1, rscratch1, _masm));
1017 }
1018 
1019 void TemplateTable::astore()
1020 {
1021   transition(vtos, vtos);
1022   __ pop_ptr(r0);
1023   locals_index(r1);
1024   __ str(r0, aaddress(r1));
1025 }
1026 
1027 void TemplateTable::wide_istore() {
1028   transition(vtos, vtos);
1029   __ pop_i();
1030   locals_index_wide(r1);
1031   __ lea(rscratch1, iaddress(r1));
1032   __ strw(r0, Address(rscratch1));
1033 }
1034 
1035 void TemplateTable::wide_lstore() {
1036   transition(vtos, vtos);
1037   __ pop_l();
1038   locals_index_wide(r1);
1039   __ str(r0, laddress(r1, rscratch1, _masm));
1040 }
1041 
1042 void TemplateTable::wide_fstore() {
1043   transition(vtos, vtos);
1044   __ pop_f();
1045   locals_index_wide(r1);
1046   __ lea(rscratch1, faddress(r1));
1047   __ strs(v0, rscratch1);
1048 }
1049 
1050 void TemplateTable::wide_dstore() {
1051   transition(vtos, vtos);
1052   __ pop_d();
1053   locals_index_wide(r1);
1054   __ strd(v0, daddress(r1, rscratch1, _masm));
1055 }
1056 
1057 void TemplateTable::wide_astore() {
1058   transition(vtos, vtos);
1059   __ pop_ptr(r0);
1060   locals_index_wide(r1);
1061   __ str(r0, aaddress(r1));
1062 }
1063 
1064 void TemplateTable::iastore() {
1065   transition(itos, vtos);
1066   __ pop_i(r1);
1067   __ pop_ptr(r3);
1068   // r0: value
1069   // r1: index
1070   // r3: array
1071   index_check(r3, r1); // prefer index in r1
1072   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
1073   __ access_store_at(T_INT, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(2)), r0, noreg, noreg);
1074 }
1075 
1076 void TemplateTable::lastore() {
1077   transition(ltos, vtos);
1078   __ pop_i(r1);
1079   __ pop_ptr(r3);
1080   // r0: value
1081   // r1: index
1082   // r3: array
1083   index_check(r3, r1); // prefer index in r1
1084   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
1085   __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(3)), r0, noreg, noreg);
1086 }
1087 
1088 void TemplateTable::fastore() {
1089   transition(ftos, vtos);
1090   __ pop_i(r1);
1091   __ pop_ptr(r3);
1092   // v0: value
1093   // r1:  index
1094   // r3:  array
1095   index_check(r3, r1); // prefer index in r1
1096   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
1097   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(2)), noreg /* ftos */, noreg, noreg);
1098 }
1099 
1100 void TemplateTable::dastore() {
1101   transition(dtos, vtos);
1102   __ pop_i(r1);
1103   __ pop_ptr(r3);
1104   // v0: value
1105   // r1:  index
1106   // r3:  array
1107   index_check(r3, r1); // prefer index in r1
1108   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
1109   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(3)), noreg /* dtos */, noreg, noreg);
1110 }
1111 
1112 void TemplateTable::aastore() {
1113   Label is_null, ok_is_subtype, done;
1114   transition(vtos, vtos);
1115   // stack: ..., array, index, value
1116   __ ldr(r0, at_tos());    // value 
1117   __ ldr(r2, at_tos_p1()); // index
1118   __ ldr(r3, at_tos_p2()); // array
1119 
1120   Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop));
1121 
1122   index_check(r3, r2);     // kills r1
1123 
1124   // FIXME: Could we remove the line below?
1125   __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); 
1126 
1127   // do array store check - check for NULL value first
1128   __ cbz(r0, is_null);
1129 
1130   Label  is_flat_array;
1131   if (ValueArrayFlatten) {
1132     __ test_flattened_array_oop(r3, r8 /*temp*/, is_flat_array);
1133   }
1134 
1135   // Move subklass into r1
1136   __ load_klass(r1, r0);
1137 
1138   // Move superklass into r0
1139   __ load_klass(r0, r3);
1140   __ ldr(r0, Address(r0, ObjArrayKlass::element_klass_offset()));
1141   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1142 
1143   // Generate subtype check.  Blows r2, r5
1144   // Superklass in r0.  Subklass in r1.
1145 
1146   __ gen_subtype_check(r1, ok_is_subtype);
1147 
1148   // Come here on failure
1149   // object is at TOS
1150   __ b(Interpreter::_throw_ArrayStoreException_entry);
1151 
1152 
1153   // Come here on success
1154   __ bind(ok_is_subtype);
1155 
1156 
1157   // Get the value we will store
1158   __ ldr(r0, at_tos());
1159   // Now store using the appropriate barrier
1160   do_oop_store(_masm, element_address, r0, IS_ARRAY);
1161   __ b(done);
1162 
1163   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1164   __ bind(is_null);
1165   __ profile_null_seen(r2);
1166 
1167   if (EnableValhalla) {
1168     Label is_null_into_value_array_npe, store_null;
1169 
1170     // No way to store null in flat array
1171     __ test_null_free_array_oop(r3, r8, is_null_into_value_array_npe); 
1172     __ b(store_null);
1173 
1174     __ bind(is_null_into_value_array_npe);
1175     __ b(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
1176 
1177     __ bind(store_null);
1178   }
1179 
1180   // Store a NULL
1181   do_oop_store(_masm, element_address, noreg, IS_ARRAY); 
1182   __ b(done);
1183 
1184   if (EnableValhalla) { 
1185      Label is_type_ok;
1186 
1187     // store non-null value
1188     __ bind(is_flat_array);
1189 
1190     // Simplistic type check...
1191     // r0 - value, r2 - index, r3 - array.
1192 
1193     // Profile the not-null value's klass.
1194     // Load value class 
1195      __ load_klass(r1, r0);
1196      __ profile_typecheck(r2, r1, r0); // blows r2, and r0
1197 
1198     // flat value array needs exact type match
1199     // is "r8 == r0" (value subclass == array element superclass)
1200 
1201     // Move element klass into r0
1202 
1203      __ load_klass(r0, r3);
1204 
1205      __ ldr(r0, Address(r0, ArrayKlass::element_klass_offset())); 
1206      __ cmp(r0, r1);
1207      __ br(Assembler::EQ, is_type_ok);
1208 
1209      __ profile_typecheck_failed(r2);
1210      __ b(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
1211 
1212      __ bind(is_type_ok);
1213 
1214     // Reload from TOS to be safe, because of profile_typecheck that blows r2 and r0.
1215     // FIXME: Should we really do it?
1216      __ ldr(r1, at_tos());  // value
1217      __ mov(r2, r3); // array, ldr(r2, at_tos_p2()); 
1218      __ ldr(r3, at_tos_p1()); // index
1219      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_store), r1, r2, r3);
1220   }
1221 
1222 
1223   // Pop stack arguments
1224   __ bind(done);
1225   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1226 }
1227 
1228 void TemplateTable::bastore()
1229 {
1230   transition(itos, vtos);
1231   __ pop_i(r1);
1232   __ pop_ptr(r3);
1233   // r0: value
1234   // r1: index
1235   // r3: array
1236   index_check(r3, r1); // prefer index in r1
1237 
1238   // Need to check whether array is boolean or byte
1239   // since both types share the bastore bytecode.
1240   __ load_klass(r2, r3);
1241   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1242   int diffbit_index = exact_log2(Klass::layout_helper_boolean_diffbit());
1243   Label L_skip;
1244   __ tbz(r2, diffbit_index, L_skip);
1245   __ andw(r0, r0, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1246   __ bind(L_skip);
1247 
1248   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
1249   __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(0)), r0, noreg, noreg);
1250 }
1251 
1252 void TemplateTable::castore()
1253 {
1254   transition(itos, vtos);
1255   __ pop_i(r1);
1256   __ pop_ptr(r3);
1257   // r0: value
1258   // r1: index
1259   // r3: array
1260   index_check(r3, r1); // prefer index in r1
1261   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
1262   __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(1)), r0, noreg, noreg);
1263 }
1264 
1265 void TemplateTable::sastore()
1266 {
1267   castore();
1268 }
1269 
1270 void TemplateTable::istore(int n)
1271 {
1272   transition(itos, vtos);
1273   __ str(r0, iaddress(n));
1274 }
1275 
1276 void TemplateTable::lstore(int n)
1277 {
1278   transition(ltos, vtos);
1279   __ str(r0, laddress(n));
1280 }
1281 
1282 void TemplateTable::fstore(int n)
1283 {
1284   transition(ftos, vtos);
1285   __ strs(v0, faddress(n));
1286 }
1287 
1288 void TemplateTable::dstore(int n)
1289 {
1290   transition(dtos, vtos);
1291   __ strd(v0, daddress(n));
1292 }
1293 
1294 void TemplateTable::astore(int n)
1295 {
1296   transition(vtos, vtos);
1297   __ pop_ptr(r0);
1298   __ str(r0, iaddress(n));
1299 }
1300 
1301 void TemplateTable::pop()
1302 {
1303   transition(vtos, vtos);
1304   __ add(esp, esp, Interpreter::stackElementSize);
1305 }
1306 
1307 void TemplateTable::pop2()
1308 {
1309   transition(vtos, vtos);
1310   __ add(esp, esp, 2 * Interpreter::stackElementSize);
1311 }
1312 
1313 void TemplateTable::dup()
1314 {
1315   transition(vtos, vtos);
1316   __ ldr(r0, Address(esp, 0));
1317   __ push(r0);
1318   // stack: ..., a, a
1319 }
1320 
1321 void TemplateTable::dup_x1()
1322 {
1323   transition(vtos, vtos);
1324   // stack: ..., a, b
1325   __ ldr(r0, at_tos());  // load b
1326   __ ldr(r2, at_tos_p1());  // load a
1327   __ str(r0, at_tos_p1());  // store b
1328   __ str(r2, at_tos());  // store a
1329   __ push(r0);                  // push b
1330   // stack: ..., b, a, b
1331 }
1332 
1333 void TemplateTable::dup_x2()
1334 {
1335   transition(vtos, vtos);
1336   // stack: ..., a, b, c
1337   __ ldr(r0, at_tos());  // load c
1338   __ ldr(r2, at_tos_p2());  // load a
1339   __ str(r0, at_tos_p2());  // store c in a
1340   __ push(r0);      // push c
1341   // stack: ..., c, b, c, c
1342   __ ldr(r0, at_tos_p2());  // load b
1343   __ str(r2, at_tos_p2());  // store a in b
1344   // stack: ..., c, a, c, c
1345   __ str(r0, at_tos_p1());  // store b in c
1346   // stack: ..., c, a, b, c
1347 }
1348 
1349 void TemplateTable::dup2()
1350 {
1351   transition(vtos, vtos);
1352   // stack: ..., a, b
1353   __ ldr(r0, at_tos_p1());  // load a
1354   __ push(r0);                  // push a
1355   __ ldr(r0, at_tos_p1());  // load b
1356   __ push(r0);                  // push b
1357   // stack: ..., a, b, a, b
1358 }
1359 
1360 void TemplateTable::dup2_x1()
1361 {
1362   transition(vtos, vtos);
1363   // stack: ..., a, b, c
1364   __ ldr(r2, at_tos());  // load c
1365   __ ldr(r0, at_tos_p1());  // load b
1366   __ push(r0);                  // push b
1367   __ push(r2);                  // push c
1368   // stack: ..., a, b, c, b, c
1369   __ str(r2, at_tos_p3());  // store c in b
1370   // stack: ..., a, c, c, b, c
1371   __ ldr(r2, at_tos_p4());  // load a
1372   __ str(r2, at_tos_p2());  // store a in 2nd c
1373   // stack: ..., a, c, a, b, c
1374   __ str(r0, at_tos_p4());  // store b in a
1375   // stack: ..., b, c, a, b, c
1376 }
1377 
1378 void TemplateTable::dup2_x2()
1379 {
1380   transition(vtos, vtos);
1381   // stack: ..., a, b, c, d
1382   __ ldr(r2, at_tos());  // load d
1383   __ ldr(r0, at_tos_p1());  // load c
1384   __ push(r0)            ;      // push c
1385   __ push(r2);                  // push d
1386   // stack: ..., a, b, c, d, c, d
1387   __ ldr(r0, at_tos_p4());  // load b
1388   __ str(r0, at_tos_p2());  // store b in d
1389   __ str(r2, at_tos_p4());  // store d in b
1390   // stack: ..., a, d, c, b, c, d
1391   __ ldr(r2, at_tos_p5());  // load a
1392   __ ldr(r0, at_tos_p3());  // load c
1393   __ str(r2, at_tos_p3());  // store a in c
1394   __ str(r0, at_tos_p5());  // store c in a
1395   // stack: ..., c, d, a, b, c, d
1396 }
1397 
1398 void TemplateTable::swap()
1399 {
1400   transition(vtos, vtos);
1401   // stack: ..., a, b
1402   __ ldr(r2, at_tos_p1());  // load a
1403   __ ldr(r0, at_tos());  // load b
1404   __ str(r2, at_tos());  // store a in b
1405   __ str(r0, at_tos_p1());  // store b in a
1406   // stack: ..., b, a
1407 }
1408 
1409 void TemplateTable::iop2(Operation op)
1410 {
1411   transition(itos, itos);
1412   // r0 <== r1 op r0
1413   __ pop_i(r1);
1414   switch (op) {
1415   case add  : __ addw(r0, r1, r0); break;
1416   case sub  : __ subw(r0, r1, r0); break;
1417   case mul  : __ mulw(r0, r1, r0); break;
1418   case _and : __ andw(r0, r1, r0); break;
1419   case _or  : __ orrw(r0, r1, r0); break;
1420   case _xor : __ eorw(r0, r1, r0); break;
1421   case shl  : __ lslvw(r0, r1, r0); break;
1422   case shr  : __ asrvw(r0, r1, r0); break;
1423   case ushr : __ lsrvw(r0, r1, r0);break;
1424   default   : ShouldNotReachHere();
1425   }
1426 }
1427 
1428 void TemplateTable::lop2(Operation op)
1429 {
1430   transition(ltos, ltos);
1431   // r0 <== r1 op r0
1432   __ pop_l(r1);
1433   switch (op) {
1434   case add  : __ add(r0, r1, r0); break;
1435   case sub  : __ sub(r0, r1, r0); break;
1436   case mul  : __ mul(r0, r1, r0); break;
1437   case _and : __ andr(r0, r1, r0); break;
1438   case _or  : __ orr(r0, r1, r0); break;
1439   case _xor : __ eor(r0, r1, r0); break;
1440   default   : ShouldNotReachHere();
1441   }
1442 }
1443 
1444 void TemplateTable::idiv()
1445 {
1446   transition(itos, itos);
1447   // explicitly check for div0
1448   Label no_div0;
1449   __ cbnzw(r0, no_div0);
1450   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1451   __ br(rscratch1);
1452   __ bind(no_div0);
1453   __ pop_i(r1);
1454   // r0 <== r1 idiv r0
1455   __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1456 }
1457 
1458 void TemplateTable::irem()
1459 {
1460   transition(itos, itos);
1461   // explicitly check for div0
1462   Label no_div0;
1463   __ cbnzw(r0, no_div0);
1464   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1465   __ br(rscratch1);
1466   __ bind(no_div0);
1467   __ pop_i(r1);
1468   // r0 <== r1 irem r0
1469   __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1470 }
1471 
1472 void TemplateTable::lmul()
1473 {
1474   transition(ltos, ltos);
1475   __ pop_l(r1);
1476   __ mul(r0, r0, r1);
1477 }
1478 
1479 void TemplateTable::ldiv()
1480 {
1481   transition(ltos, ltos);
1482   // explicitly check for div0
1483   Label no_div0;
1484   __ cbnz(r0, no_div0);
1485   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1486   __ br(rscratch1);
1487   __ bind(no_div0);
1488   __ pop_l(r1);
1489   // r0 <== r1 ldiv r0
1490   __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1491 }
1492 
1493 void TemplateTable::lrem()
1494 {
1495   transition(ltos, ltos);
1496   // explicitly check for div0
1497   Label no_div0;
1498   __ cbnz(r0, no_div0);
1499   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1500   __ br(rscratch1);
1501   __ bind(no_div0);
1502   __ pop_l(r1);
1503   // r0 <== r1 lrem r0
1504   __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1505 }
1506 
1507 void TemplateTable::lshl()
1508 {
1509   transition(itos, ltos);
1510   // shift count is in r0
1511   __ pop_l(r1);
1512   __ lslv(r0, r1, r0);
1513 }
1514 
1515 void TemplateTable::lshr()
1516 {
1517   transition(itos, ltos);
1518   // shift count is in r0
1519   __ pop_l(r1);
1520   __ asrv(r0, r1, r0);
1521 }
1522 
1523 void TemplateTable::lushr()
1524 {
1525   transition(itos, ltos);
1526   // shift count is in r0
1527   __ pop_l(r1);
1528   __ lsrv(r0, r1, r0);
1529 }
1530 
1531 void TemplateTable::fop2(Operation op)
1532 {
1533   transition(ftos, ftos);
1534   switch (op) {
1535   case add:
1536     // n.b. use ldrd because this is a 64 bit slot
1537     __ pop_f(v1);
1538     __ fadds(v0, v1, v0);
1539     break;
1540   case sub:
1541     __ pop_f(v1);
1542     __ fsubs(v0, v1, v0);
1543     break;
1544   case mul:
1545     __ pop_f(v1);
1546     __ fmuls(v0, v1, v0);
1547     break;
1548   case div:
1549     __ pop_f(v1);
1550     __ fdivs(v0, v1, v0);
1551     break;
1552   case rem:
1553     __ fmovs(v1, v0);
1554     __ pop_f(v0);
1555     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::frem),
1556                          0, 2, MacroAssembler::ret_type_float);
1557     break;
1558   default:
1559     ShouldNotReachHere();
1560     break;
1561   }
1562 }
1563 
1564 void TemplateTable::dop2(Operation op)
1565 {
1566   transition(dtos, dtos);
1567   switch (op) {
1568   case add:
1569     // n.b. use ldrd because this is a 64 bit slot
1570     __ pop_d(v1);
1571     __ faddd(v0, v1, v0);
1572     break;
1573   case sub:
1574     __ pop_d(v1);
1575     __ fsubd(v0, v1, v0);
1576     break;
1577   case mul:
1578     __ pop_d(v1);
1579     __ fmuld(v0, v1, v0);
1580     break;
1581   case div:
1582     __ pop_d(v1);
1583     __ fdivd(v0, v1, v0);
1584     break;
1585   case rem:
1586     __ fmovd(v1, v0);
1587     __ pop_d(v0);
1588     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::drem),
1589                          0, 2, MacroAssembler::ret_type_double);
1590     break;
1591   default:
1592     ShouldNotReachHere();
1593     break;
1594   }
1595 }
1596 
1597 void TemplateTable::ineg()
1598 {
1599   transition(itos, itos);
1600   __ negw(r0, r0);
1601 
1602 }
1603 
1604 void TemplateTable::lneg()
1605 {
1606   transition(ltos, ltos);
1607   __ neg(r0, r0);
1608 }
1609 
1610 void TemplateTable::fneg()
1611 {
1612   transition(ftos, ftos);
1613   __ fnegs(v0, v0);
1614 }
1615 
1616 void TemplateTable::dneg()
1617 {
1618   transition(dtos, dtos);
1619   __ fnegd(v0, v0);
1620 }
1621 
1622 void TemplateTable::iinc()
1623 {
1624   transition(vtos, vtos);
1625   __ load_signed_byte(r1, at_bcp(2)); // get constant
1626   locals_index(r2);
1627   __ ldr(r0, iaddress(r2));
1628   __ addw(r0, r0, r1);
1629   __ str(r0, iaddress(r2));
1630 }
1631 
1632 void TemplateTable::wide_iinc()
1633 {
1634   transition(vtos, vtos);
1635   // __ mov(r1, zr);
1636   __ ldrw(r1, at_bcp(2)); // get constant and index
1637   __ rev16(r1, r1);
1638   __ ubfx(r2, r1, 0, 16);
1639   __ neg(r2, r2);
1640   __ sbfx(r1, r1, 16, 16);
1641   __ ldr(r0, iaddress(r2));
1642   __ addw(r0, r0, r1);
1643   __ str(r0, iaddress(r2));
1644 }
1645 
1646 void TemplateTable::convert()
1647 {
1648   // Checking
1649 #ifdef ASSERT
1650   {
1651     TosState tos_in  = ilgl;
1652     TosState tos_out = ilgl;
1653     switch (bytecode()) {
1654     case Bytecodes::_i2l: // fall through
1655     case Bytecodes::_i2f: // fall through
1656     case Bytecodes::_i2d: // fall through
1657     case Bytecodes::_i2b: // fall through
1658     case Bytecodes::_i2c: // fall through
1659     case Bytecodes::_i2s: tos_in = itos; break;
1660     case Bytecodes::_l2i: // fall through
1661     case Bytecodes::_l2f: // fall through
1662     case Bytecodes::_l2d: tos_in = ltos; break;
1663     case Bytecodes::_f2i: // fall through
1664     case Bytecodes::_f2l: // fall through
1665     case Bytecodes::_f2d: tos_in = ftos; break;
1666     case Bytecodes::_d2i: // fall through
1667     case Bytecodes::_d2l: // fall through
1668     case Bytecodes::_d2f: tos_in = dtos; break;
1669     default             : ShouldNotReachHere();
1670     }
1671     switch (bytecode()) {
1672     case Bytecodes::_l2i: // fall through
1673     case Bytecodes::_f2i: // fall through
1674     case Bytecodes::_d2i: // fall through
1675     case Bytecodes::_i2b: // fall through
1676     case Bytecodes::_i2c: // fall through
1677     case Bytecodes::_i2s: tos_out = itos; break;
1678     case Bytecodes::_i2l: // fall through
1679     case Bytecodes::_f2l: // fall through
1680     case Bytecodes::_d2l: tos_out = ltos; break;
1681     case Bytecodes::_i2f: // fall through
1682     case Bytecodes::_l2f: // fall through
1683     case Bytecodes::_d2f: tos_out = ftos; break;
1684     case Bytecodes::_i2d: // fall through
1685     case Bytecodes::_l2d: // fall through
1686     case Bytecodes::_f2d: tos_out = dtos; break;
1687     default             : ShouldNotReachHere();
1688     }
1689     transition(tos_in, tos_out);
1690   }
1691 #endif // ASSERT
1692   // static const int64_t is_nan = 0x8000000000000000L;
1693 
1694   // Conversion
1695   switch (bytecode()) {
1696   case Bytecodes::_i2l:
1697     __ sxtw(r0, r0);
1698     break;
1699   case Bytecodes::_i2f:
1700     __ scvtfws(v0, r0);
1701     break;
1702   case Bytecodes::_i2d:
1703     __ scvtfwd(v0, r0);
1704     break;
1705   case Bytecodes::_i2b:
1706     __ sxtbw(r0, r0);
1707     break;
1708   case Bytecodes::_i2c:
1709     __ uxthw(r0, r0);
1710     break;
1711   case Bytecodes::_i2s:
1712     __ sxthw(r0, r0);
1713     break;
1714   case Bytecodes::_l2i:
1715     __ uxtw(r0, r0);
1716     break;
1717   case Bytecodes::_l2f:
1718     __ scvtfs(v0, r0);
1719     break;
1720   case Bytecodes::_l2d:
1721     __ scvtfd(v0, r0);
1722     break;
1723   case Bytecodes::_f2i:
1724   {
1725     Label L_Okay;
1726     __ clear_fpsr();
1727     __ fcvtzsw(r0, v0);
1728     __ get_fpsr(r1);
1729     __ cbzw(r1, L_Okay);
1730     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2i),
1731                          0, 1, MacroAssembler::ret_type_integral);
1732     __ bind(L_Okay);
1733   }
1734     break;
1735   case Bytecodes::_f2l:
1736   {
1737     Label L_Okay;
1738     __ clear_fpsr();
1739     __ fcvtzs(r0, v0);
1740     __ get_fpsr(r1);
1741     __ cbzw(r1, L_Okay);
1742     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2l),
1743                          0, 1, MacroAssembler::ret_type_integral);
1744     __ bind(L_Okay);
1745   }
1746     break;
1747   case Bytecodes::_f2d:
1748     __ fcvts(v0, v0);
1749     break;
1750   case Bytecodes::_d2i:
1751   {
1752     Label L_Okay;
1753     __ clear_fpsr();
1754     __ fcvtzdw(r0, v0);
1755     __ get_fpsr(r1);
1756     __ cbzw(r1, L_Okay);
1757     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2i),
1758                          0, 1, MacroAssembler::ret_type_integral);
1759     __ bind(L_Okay);
1760   }
1761     break;
1762   case Bytecodes::_d2l:
1763   {
1764     Label L_Okay;
1765     __ clear_fpsr();
1766     __ fcvtzd(r0, v0);
1767     __ get_fpsr(r1);
1768     __ cbzw(r1, L_Okay);
1769     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2l),
1770                          0, 1, MacroAssembler::ret_type_integral);
1771     __ bind(L_Okay);
1772   }
1773     break;
1774   case Bytecodes::_d2f:
1775     __ fcvtd(v0, v0);
1776     break;
1777   default:
1778     ShouldNotReachHere();
1779   }
1780 }
1781 
1782 void TemplateTable::lcmp()
1783 {
1784   transition(ltos, itos);
1785   Label done;
1786   __ pop_l(r1);
1787   __ cmp(r1, r0);
1788   __ mov(r0, (u_int64_t)-1L);
1789   __ br(Assembler::LT, done);
1790   // __ mov(r0, 1UL);
1791   // __ csel(r0, r0, zr, Assembler::NE);
1792   // and here is a faster way
1793   __ csinc(r0, zr, zr, Assembler::EQ);
1794   __ bind(done);
1795 }
1796 
1797 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1798 {
1799   Label done;
1800   if (is_float) {
1801     // XXX get rid of pop here, use ... reg, mem32
1802     __ pop_f(v1);
1803     __ fcmps(v1, v0);
1804   } else {
1805     // XXX get rid of pop here, use ... reg, mem64
1806     __ pop_d(v1);
1807     __ fcmpd(v1, v0);
1808   }
1809   if (unordered_result < 0) {
1810     // we want -1 for unordered or less than, 0 for equal and 1 for
1811     // greater than.
1812     __ mov(r0, (u_int64_t)-1L);
1813     // for FP LT tests less than or unordered
1814     __ br(Assembler::LT, done);
1815     // install 0 for EQ otherwise 1
1816     __ csinc(r0, zr, zr, Assembler::EQ);
1817   } else {
1818     // we want -1 for less than, 0 for equal and 1 for unordered or
1819     // greater than.
1820     __ mov(r0, 1L);
1821     // for FP HI tests greater than or unordered
1822     __ br(Assembler::HI, done);
1823     // install 0 for EQ otherwise ~0
1824     __ csinv(r0, zr, zr, Assembler::EQ);
1825 
1826   }
1827   __ bind(done);
1828 }
1829 
1830 void TemplateTable::branch(bool is_jsr, bool is_wide)
1831 {
1832   // We might be moving to a safepoint.  The thread which calls
1833   // Interpreter::notice_safepoints() will effectively flush its cache
1834   // when it makes a system call, but we need to do something to
1835   // ensure that we see the changed dispatch table.
1836   __ membar(MacroAssembler::LoadLoad);
1837 
1838   __ profile_taken_branch(r0, r1);
1839   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1840                              InvocationCounter::counter_offset();
1841   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1842                               InvocationCounter::counter_offset();
1843 
1844   // load branch displacement
1845   if (!is_wide) {
1846     __ ldrh(r2, at_bcp(1));
1847     __ rev16(r2, r2);
1848     // sign extend the 16 bit value in r2
1849     __ sbfm(r2, r2, 0, 15);
1850   } else {
1851     __ ldrw(r2, at_bcp(1));
1852     __ revw(r2, r2);
1853     // sign extend the 32 bit value in r2
1854     __ sbfm(r2, r2, 0, 31);
1855   }
1856 
1857   // Handle all the JSR stuff here, then exit.
1858   // It's much shorter and cleaner than intermingling with the non-JSR
1859   // normal-branch stuff occurring below.
1860 
1861   if (is_jsr) {
1862     // Pre-load the next target bytecode into rscratch1
1863     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1864     // compute return address as bci
1865     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1866     __ add(rscratch2, rscratch2,
1867            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1868     __ sub(r1, rbcp, rscratch2);
1869     __ push_i(r1);
1870     // Adjust the bcp by the 16-bit displacement in r2
1871     __ add(rbcp, rbcp, r2);
1872     __ dispatch_only(vtos, /*generate_poll*/true);
1873     return;
1874   }
1875 
1876   // Normal (non-jsr) branch handling
1877 
1878   // Adjust the bcp by the displacement in r2
1879   __ add(rbcp, rbcp, r2);
1880 
1881   assert(UseLoopCounter || !UseOnStackReplacement,
1882          "on-stack-replacement requires loop counters");
1883   Label backedge_counter_overflow;
1884   Label profile_method;
1885   Label dispatch;
1886   if (UseLoopCounter) {
1887     // increment backedge counter for backward branches
1888     // r0: MDO
1889     // w1: MDO bumped taken-count
1890     // r2: target offset
1891     __ cmp(r2, zr);
1892     __ br(Assembler::GT, dispatch); // count only if backward branch
1893 
1894     // ECN: FIXME: This code smells
1895     // check if MethodCounters exists
1896     Label has_counters;
1897     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1898     __ cbnz(rscratch1, has_counters);
1899     __ push(r0);
1900     __ push(r1);
1901     __ push(r2);
1902     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1903             InterpreterRuntime::build_method_counters), rmethod);
1904     __ pop(r2);
1905     __ pop(r1);
1906     __ pop(r0);
1907     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1908     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1909     __ bind(has_counters);
1910 
1911     if (TieredCompilation) {
1912       Label no_mdo;
1913       int increment = InvocationCounter::count_increment;
1914       if (ProfileInterpreter) {
1915         // Are we profiling?
1916         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1917         __ cbz(r1, no_mdo);
1918         // Increment the MDO backedge counter
1919         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1920                                            in_bytes(InvocationCounter::counter_offset()));
1921         const Address mask(r1, in_bytes(MethodData::backedge_mask_offset()));
1922         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1923                                    r0, rscratch1, false, Assembler::EQ,
1924                                    UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1925         __ b(dispatch);
1926       }
1927       __ bind(no_mdo);
1928       // Increment backedge counter in MethodCounters*
1929       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1930       const Address mask(rscratch1, in_bytes(MethodCounters::backedge_mask_offset()));
1931       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1932                                  r0, rscratch2, false, Assembler::EQ,
1933                                  UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1934     } else { // not TieredCompilation
1935       // increment counter
1936       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1937       __ ldrw(r0, Address(rscratch2, be_offset));        // load backedge counter
1938       __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1939       __ strw(rscratch1, Address(rscratch2, be_offset));        // store counter
1940 
1941       __ ldrw(r0, Address(rscratch2, inv_offset));    // load invocation counter
1942       __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1943       __ addw(r0, r0, rscratch1);        // add both counters
1944 
1945       if (ProfileInterpreter) {
1946         // Test to see if we should create a method data oop
1947         __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_profile_limit_offset())));
1948         __ cmpw(r0, rscratch1);
1949         __ br(Assembler::LT, dispatch);
1950 
1951         // if no method data exists, go to profile method
1952         __ test_method_data_pointer(r0, profile_method);
1953 
1954         if (UseOnStackReplacement) {
1955           // check for overflow against w1 which is the MDO taken count
1956           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1957           __ cmpw(r1, rscratch1);
1958           __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1959 
1960           // When ProfileInterpreter is on, the backedge_count comes
1961           // from the MethodData*, which value does not get reset on
1962           // the call to frequency_counter_overflow().  To avoid
1963           // excessive calls to the overflow routine while the method is
1964           // being compiled, add a second test to make sure the overflow
1965           // function is called only once every overflow_frequency.
1966           const int overflow_frequency = 1024;
1967           __ andsw(r1, r1, overflow_frequency - 1);
1968           __ br(Assembler::EQ, backedge_counter_overflow);
1969 
1970         }
1971       } else {
1972         if (UseOnStackReplacement) {
1973           // check for overflow against w0, which is the sum of the
1974           // counters
1975           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1976           __ cmpw(r0, rscratch1);
1977           __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1978         }
1979       }
1980     }
1981     __ bind(dispatch);
1982   }
1983 
1984   // Pre-load the next target bytecode into rscratch1
1985   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1986 
1987   // continue with the bytecode @ target
1988   // rscratch1: target bytecode
1989   // rbcp: target bcp
1990   __ dispatch_only(vtos, /*generate_poll*/true);
1991 
1992   if (UseLoopCounter) {
1993     if (ProfileInterpreter) {
1994       // Out-of-line code to allocate method data oop.
1995       __ bind(profile_method);
1996       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1997       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1998       __ set_method_data_pointer_for_bcp();
1999       __ b(dispatch);
2000     }
2001 
2002     if (UseOnStackReplacement) {
2003       // invocation counter overflow
2004       __ bind(backedge_counter_overflow);
2005       __ neg(r2, r2);
2006       __ add(r2, r2, rbcp);     // branch bcp
2007       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
2008       __ call_VM(noreg,
2009                  CAST_FROM_FN_PTR(address,
2010                                   InterpreterRuntime::frequency_counter_overflow),
2011                  r2);
2012       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
2013 
2014       // r0: osr nmethod (osr ok) or NULL (osr not possible)
2015       // w1: target bytecode
2016       // r2: scratch
2017       __ cbz(r0, dispatch);     // test result -- no osr if null
2018       // nmethod may have been invalidated (VM may block upon call_VM return)
2019       __ ldrb(r2, Address(r0, nmethod::state_offset()));
2020       if (nmethod::in_use != 0)
2021         __ sub(r2, r2, nmethod::in_use);
2022       __ cbnz(r2, dispatch);
2023 
2024       // We have the address of an on stack replacement routine in r0
2025       // We need to prepare to execute the OSR method. First we must
2026       // migrate the locals and monitors off of the stack.
2027 
2028       __ mov(r19, r0);                             // save the nmethod
2029 
2030       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2031 
2032       // r0 is OSR buffer, move it to expected parameter location
2033       __ mov(j_rarg0, r0);
2034 
2035       // remove activation
2036       // get sender esp
2037       __ ldr(esp,
2038           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
2039       // remove frame anchor
2040       __ leave();
2041       // Ensure compiled code always sees stack at proper alignment
2042       __ andr(sp, esp, -16);
2043 
2044       // and begin the OSR nmethod
2045       __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
2046       __ br(rscratch1);
2047     }
2048   }
2049 }
2050 
2051 
2052 void TemplateTable::if_0cmp(Condition cc)
2053 {
2054   transition(itos, vtos);
2055   // assume branch is more often taken than not (loops use backward branches)
2056   Label not_taken;
2057   if (cc == equal)
2058     __ cbnzw(r0, not_taken);
2059   else if (cc == not_equal)
2060     __ cbzw(r0, not_taken);
2061   else {
2062     __ andsw(zr, r0, r0);
2063     __ br(j_not(cc), not_taken);
2064   }
2065 
2066   branch(false, false);
2067   __ bind(not_taken);
2068   __ profile_not_taken_branch(r0);
2069 }
2070 
2071 void TemplateTable::if_icmp(Condition cc)
2072 {
2073   transition(itos, vtos);
2074   // assume branch is more often taken than not (loops use backward branches)
2075   Label not_taken;
2076   __ pop_i(r1);
2077   __ cmpw(r1, r0, Assembler::LSL);
2078   __ br(j_not(cc), not_taken);
2079   branch(false, false);
2080   __ bind(not_taken);
2081   __ profile_not_taken_branch(r0);
2082 }
2083 
2084 void TemplateTable::if_nullcmp(Condition cc)
2085 {
2086   transition(atos, vtos);
2087   // assume branch is more often taken than not (loops use backward branches)
2088   Label not_taken;
2089   if (cc == equal)
2090     __ cbnz(r0, not_taken);
2091   else
2092     __ cbz(r0, not_taken);
2093   branch(false, false);
2094   __ bind(not_taken);
2095   __ profile_not_taken_branch(r0);
2096 }
2097 
2098 void TemplateTable::if_acmp(Condition cc) {
2099   transition(atos, vtos);
2100   // assume branch is more often taken than not (loops use backward branches)
2101   Label taken, not_taken;
2102   __ pop_ptr(r1);
2103 
2104   Register is_value_mask = rscratch1;
2105   __ mov(is_value_mask, markOopDesc::always_locked_pattern);
2106 
2107   if (EnableValhalla) {
2108     __ cmp(r1, r0);
2109     __ br(Assembler::EQ, (cc == equal) ? taken : not_taken);
2110 
2111     // might be substitutable, test if either r0 or r1 is null
2112     __ andr(r2, r0, r1);
2113     __ cbz(r2, (cc == equal) ? not_taken : taken);
2114 
2115     // and both are values ?
2116     __ ldr(r2, Address(r1, oopDesc::mark_offset_in_bytes()));
2117     __ andr(r2, r2, is_value_mask);
2118     __ ldr(r4, Address(r0, oopDesc::mark_offset_in_bytes()));
2119     __ andr(r4, r4, is_value_mask);
2120     __ andr(r2, r2, r4);
2121     __ cmp(r2,  is_value_mask);
2122     __ br(Assembler::NE, (cc == equal) ? not_taken : taken);
2123 
2124     // same value klass ?
2125     __ load_metadata(r2, r1);
2126     __ load_metadata(r4, r0);
2127     __ cmp(r2, r4);
2128     __ br(Assembler::NE, (cc == equal) ? not_taken : taken);
2129 
2130     // Know both are the same type, let's test for substitutability...
2131     if (cc == equal) {
2132       invoke_is_substitutable(r0, r1, taken, not_taken);
2133     } else {
2134       invoke_is_substitutable(r0, r1, not_taken, taken);
2135     }
2136     __ stop("Not reachable");
2137   }
2138 











2139   __ cmpoop(r1, r0);










2140   __ br(j_not(cc), not_taken);
2141   __ bind(taken);
2142   branch(false, false);
2143   __ bind(not_taken);
2144   __ profile_not_taken_branch(r0);
2145 }
2146 
2147 void TemplateTable::invoke_is_substitutable(Register aobj, Register bobj,
2148                                             Label& is_subst, Label& not_subst) {
2149 
2150   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::is_substitutable), aobj, bobj);
2151   // Restored... r0 answer, jmp to outcome...
2152   __ cbz(r0, not_subst);
2153   __ b(is_subst);
2154 }
2155 
2156 
2157 void TemplateTable::ret() {
2158   transition(vtos, vtos);
2159   // We might be moving to a safepoint.  The thread which calls
2160   // Interpreter::notice_safepoints() will effectively flush its cache
2161   // when it makes a system call, but we need to do something to
2162   // ensure that we see the changed dispatch table.
2163   __ membar(MacroAssembler::LoadLoad);
2164 
2165   locals_index(r1);
2166   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2167   __ profile_ret(r1, r2);
2168   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2169   __ lea(rbcp, Address(rbcp, r1));
2170   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2171   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2172 }
2173 
2174 void TemplateTable::wide_ret() {
2175   transition(vtos, vtos);
2176   locals_index_wide(r1);
2177   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2178   __ profile_ret(r1, r2);
2179   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2180   __ lea(rbcp, Address(rbcp, r1));
2181   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2182   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2183 }
2184 
2185 
2186 void TemplateTable::tableswitch() {
2187   Label default_case, continue_execution;
2188   transition(itos, vtos);
2189   // align rbcp
2190   __ lea(r1, at_bcp(BytesPerInt));
2191   __ andr(r1, r1, -BytesPerInt);
2192   // load lo & hi
2193   __ ldrw(r2, Address(r1, BytesPerInt));
2194   __ ldrw(r3, Address(r1, 2 * BytesPerInt));
2195   __ rev32(r2, r2);
2196   __ rev32(r3, r3);
2197   // check against lo & hi
2198   __ cmpw(r0, r2);
2199   __ br(Assembler::LT, default_case);
2200   __ cmpw(r0, r3);
2201   __ br(Assembler::GT, default_case);
2202   // lookup dispatch offset
2203   __ subw(r0, r0, r2);
2204   __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2205   __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2206   __ profile_switch_case(r0, r1, r2);
2207   // continue execution
2208   __ bind(continue_execution);
2209   __ rev32(r3, r3);
2210   __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2211   __ add(rbcp, rbcp, r3, ext::sxtw);
2212   __ dispatch_only(vtos, /*generate_poll*/true);
2213   // handle default
2214   __ bind(default_case);
2215   __ profile_switch_default(r0);
2216   __ ldrw(r3, Address(r1, 0));
2217   __ b(continue_execution);
2218 }
2219 
2220 void TemplateTable::lookupswitch() {
2221   transition(itos, itos);
2222   __ stop("lookupswitch bytecode should have been rewritten");
2223 }
2224 
2225 void TemplateTable::fast_linearswitch() {
2226   transition(itos, vtos);
2227   Label loop_entry, loop, found, continue_execution;
2228   // bswap r0 so we can avoid bswapping the table entries
2229   __ rev32(r0, r0);
2230   // align rbcp
2231   __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2232                                     // this instruction (change offsets
2233                                     // below)
2234   __ andr(r19, r19, -BytesPerInt);
2235   // set counter
2236   __ ldrw(r1, Address(r19, BytesPerInt));
2237   __ rev32(r1, r1);
2238   __ b(loop_entry);
2239   // table search
2240   __ bind(loop);
2241   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2242   __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2243   __ cmpw(r0, rscratch1);
2244   __ br(Assembler::EQ, found);
2245   __ bind(loop_entry);
2246   __ subs(r1, r1, 1);
2247   __ br(Assembler::PL, loop);
2248   // default case
2249   __ profile_switch_default(r0);
2250   __ ldrw(r3, Address(r19, 0));
2251   __ b(continue_execution);
2252   // entry found -> get offset
2253   __ bind(found);
2254   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2255   __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2256   __ profile_switch_case(r1, r0, r19);
2257   // continue execution
2258   __ bind(continue_execution);
2259   __ rev32(r3, r3);
2260   __ add(rbcp, rbcp, r3, ext::sxtw);
2261   __ ldrb(rscratch1, Address(rbcp, 0));
2262   __ dispatch_only(vtos, /*generate_poll*/true);
2263 }
2264 
2265 void TemplateTable::fast_binaryswitch() {
2266   transition(itos, vtos);
2267   // Implementation using the following core algorithm:
2268   //
2269   // int binary_search(int key, LookupswitchPair* array, int n) {
2270   //   // Binary search according to "Methodik des Programmierens" by
2271   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2272   //   int i = 0;
2273   //   int j = n;
2274   //   while (i+1 < j) {
2275   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2276   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2277   //     // where a stands for the array and assuming that the (inexisting)
2278   //     // element a[n] is infinitely big.
2279   //     int h = (i + j) >> 1;
2280   //     // i < h < j
2281   //     if (key < array[h].fast_match()) {
2282   //       j = h;
2283   //     } else {
2284   //       i = h;
2285   //     }
2286   //   }
2287   //   // R: a[i] <= key < a[i+1] or Q
2288   //   // (i.e., if key is within array, i is the correct index)
2289   //   return i;
2290   // }
2291 
2292   // Register allocation
2293   const Register key   = r0; // already set (tosca)
2294   const Register array = r1;
2295   const Register i     = r2;
2296   const Register j     = r3;
2297   const Register h     = rscratch1;
2298   const Register temp  = rscratch2;
2299 
2300   // Find array start
2301   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2302                                           // get rid of this
2303                                           // instruction (change
2304                                           // offsets below)
2305   __ andr(array, array, -BytesPerInt);
2306 
2307   // Initialize i & j
2308   __ mov(i, 0);                            // i = 0;
2309   __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2310 
2311   // Convert j into native byteordering
2312   __ rev32(j, j);
2313 
2314   // And start
2315   Label entry;
2316   __ b(entry);
2317 
2318   // binary search loop
2319   {
2320     Label loop;
2321     __ bind(loop);
2322     // int h = (i + j) >> 1;
2323     __ addw(h, i, j);                           // h = i + j;
2324     __ lsrw(h, h, 1);                                   // h = (i + j) >> 1;
2325     // if (key < array[h].fast_match()) {
2326     //   j = h;
2327     // } else {
2328     //   i = h;
2329     // }
2330     // Convert array[h].match to native byte-ordering before compare
2331     __ ldr(temp, Address(array, h, Address::lsl(3)));
2332     __ rev32(temp, temp);
2333     __ cmpw(key, temp);
2334     // j = h if (key <  array[h].fast_match())
2335     __ csel(j, h, j, Assembler::LT);
2336     // i = h if (key >= array[h].fast_match())
2337     __ csel(i, h, i, Assembler::GE);
2338     // while (i+1 < j)
2339     __ bind(entry);
2340     __ addw(h, i, 1);          // i+1
2341     __ cmpw(h, j);             // i+1 < j
2342     __ br(Assembler::LT, loop);
2343   }
2344 
2345   // end of binary search, result index is i (must check again!)
2346   Label default_case;
2347   // Convert array[i].match to native byte-ordering before compare
2348   __ ldr(temp, Address(array, i, Address::lsl(3)));
2349   __ rev32(temp, temp);
2350   __ cmpw(key, temp);
2351   __ br(Assembler::NE, default_case);
2352 
2353   // entry found -> j = offset
2354   __ add(j, array, i, ext::uxtx, 3);
2355   __ ldrw(j, Address(j, BytesPerInt));
2356   __ profile_switch_case(i, key, array);
2357   __ rev32(j, j);
2358   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2359   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2360   __ dispatch_only(vtos, /*generate_poll*/true);
2361 
2362   // default case -> j = default offset
2363   __ bind(default_case);
2364   __ profile_switch_default(i);
2365   __ ldrw(j, Address(array, -2 * BytesPerInt));
2366   __ rev32(j, j);
2367   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2368   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2369   __ dispatch_only(vtos, /*generate_poll*/true);
2370 }
2371 
2372 
2373 void TemplateTable::_return(TosState state)
2374 {
2375   transition(state, state);
2376   assert(_desc->calls_vm(),
2377          "inconsistent calls_vm information"); // call in remove_activation
2378 
2379   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2380     assert(state == vtos, "only valid state");
2381 
2382     __ ldr(c_rarg1, aaddress(0));
2383     __ load_klass(r3, c_rarg1);
2384     __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2385     Label skip_register_finalizer;
2386     __ tbz(r3, exact_log2(JVM_ACC_HAS_FINALIZER), skip_register_finalizer);
2387 
2388     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2389 
2390     __ bind(skip_register_finalizer);
2391   }
2392 
2393   // Issue a StoreStore barrier after all stores but before return
2394   // from any constructor for any class with a final field.  We don't
2395   // know if this is a finalizer, so we always do so.
2396   if (_desc->bytecode() == Bytecodes::_return)
2397     __ membar(MacroAssembler::StoreStore);
2398 
2399   // Narrow result if state is itos but result type is smaller.
2400   // Need to narrow in the return bytecode rather than in generate_return_entry
2401   // since compiled code callers expect the result to already be narrowed.
2402   if (state == itos) {
2403     __ narrow(r0);
2404   }
2405 
2406   __ remove_activation(state); 
2407   __ ret(lr);
2408 }
2409 
2410 // ----------------------------------------------------------------------------
2411 // Volatile variables demand their effects be made known to all CPU's
2412 // in order.  Store buffers on most chips allow reads & writes to
2413 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2414 // without some kind of memory barrier (i.e., it's not sufficient that
2415 // the interpreter does not reorder volatile references, the hardware
2416 // also must not reorder them).
2417 //
2418 // According to the new Java Memory Model (JMM):
2419 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2420 //     writes act as aquire & release, so:
2421 // (2) A read cannot let unrelated NON-volatile memory refs that
2422 //     happen after the read float up to before the read.  It's OK for
2423 //     non-volatile memory refs that happen before the volatile read to
2424 //     float down below it.
2425 // (3) Similar a volatile write cannot let unrelated NON-volatile
2426 //     memory refs that happen BEFORE the write float down to after the
2427 //     write.  It's OK for non-volatile memory refs that happen after the
2428 //     volatile write to float up before it.
2429 //
2430 // We only put in barriers around volatile refs (they are expensive),
2431 // not _between_ memory refs (that would require us to track the
2432 // flavor of the previous memory refs).  Requirements (2) and (3)
2433 // require some barriers before volatile stores and after volatile
2434 // loads.  These nearly cover requirement (1) but miss the
2435 // volatile-store-volatile-load case.  This final case is placed after
2436 // volatile-stores although it could just as well go before
2437 // volatile-loads.
2438 
2439 void TemplateTable::resolve_cache_and_index(int byte_no,
2440                                             Register Rcache,
2441                                             Register index,
2442                                             size_t index_size) {
2443   const Register temp = r19;
2444   assert_different_registers(Rcache, index, temp);
2445 
2446   Label resolved;
2447 
2448   Bytecodes::Code code = bytecode();
2449   switch (code) {
2450   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2451   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2452   default: break;
2453   }
2454 
2455   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2456   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2457   __ subs(zr, temp, (int) code);  // have we resolved this bytecode?
2458   __ br(Assembler::EQ, resolved);
2459 
2460   // resolve first time through
2461   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2462   __ mov(temp, (int) code);
2463   __ call_VM(noreg, entry, temp);
2464 
2465   // Update registers with resolved info
2466   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2467   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2468   // so all clients ofthis method must be modified accordingly
2469   __ bind(resolved);
2470 }
2471 
2472 // The Rcache and index registers must be set before call
2473 // n.b unlike x86 cache already includes the index offset
2474 void TemplateTable::load_field_cp_cache_entry(Register obj,
2475                                               Register cache,
2476                                               Register index,
2477                                               Register off,
2478                                               Register flags,
2479                                               bool is_static = false) {
2480   assert_different_registers(cache, index, flags, off);
2481 
2482   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2483   // Field offset
2484   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2485                                           ConstantPoolCacheEntry::f2_offset())));
2486   // Flags
2487   __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2488                                            ConstantPoolCacheEntry::flags_offset())));
2489 
2490   // klass overwrite register
2491   if (is_static) {
2492     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2493                                         ConstantPoolCacheEntry::f1_offset())));
2494     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2495     __ ldr(obj, Address(obj, mirror_offset));
2496     __ resolve_oop_handle(obj);
2497   }
2498 }
2499 
2500 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2501                                                Register method,
2502                                                Register itable_index,
2503                                                Register flags,
2504                                                bool is_invokevirtual,
2505                                                bool is_invokevfinal, /*unused*/
2506                                                bool is_invokedynamic) {
2507   // setup registers
2508   const Register cache = rscratch2;
2509   const Register index = r4;
2510   assert_different_registers(method, flags);
2511   assert_different_registers(method, cache, index);
2512   assert_different_registers(itable_index, flags);
2513   assert_different_registers(itable_index, cache, index);
2514   // determine constant pool cache field offsets
2515   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2516   const int method_offset = in_bytes(
2517     ConstantPoolCache::base_offset() +
2518       (is_invokevirtual
2519        ? ConstantPoolCacheEntry::f2_offset()
2520        : ConstantPoolCacheEntry::f1_offset()));
2521   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2522                                     ConstantPoolCacheEntry::flags_offset());
2523   // access constant pool cache fields
2524   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2525                                     ConstantPoolCacheEntry::f2_offset());
2526 
2527   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2528   resolve_cache_and_index(byte_no, cache, index, index_size);
2529   __ ldr(method, Address(cache, method_offset));
2530 
2531   if (itable_index != noreg) {
2532     __ ldr(itable_index, Address(cache, index_offset));
2533   }
2534   __ ldrw(flags, Address(cache, flags_offset));
2535 }
2536 
2537 
2538 // The registers cache and index expected to be set before call.
2539 // Correct values of the cache and index registers are preserved.
2540 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2541                                             bool is_static, bool has_tos) {
2542   // do the JVMTI work here to avoid disturbing the register state below
2543   // We use c_rarg registers here because we want to use the register used in
2544   // the call to the VM
2545   if (JvmtiExport::can_post_field_access()) {
2546     // Check to see if a field access watch has been set before we
2547     // take the time to call into the VM.
2548     Label L1;
2549     assert_different_registers(cache, index, r0);
2550     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2551     __ ldrw(r0, Address(rscratch1));
2552     __ cbzw(r0, L1);
2553 
2554     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2555     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2556 
2557     if (is_static) {
2558       __ mov(c_rarg1, zr); // NULL object reference
2559     } else {
2560       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2561       __ verify_oop(c_rarg1);
2562     }
2563     // c_rarg1: object pointer or NULL
2564     // c_rarg2: cache entry pointer
2565     // c_rarg3: jvalue object on the stack
2566     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2567                                        InterpreterRuntime::post_field_access),
2568                c_rarg1, c_rarg2, c_rarg3);
2569     __ get_cache_and_index_at_bcp(cache, index, 1);
2570     __ bind(L1);
2571   }
2572 }
2573 
2574 void TemplateTable::pop_and_check_object(Register r)
2575 {
2576   __ pop_ptr(r);
2577   __ null_check(r);  // for field access must check obj.
2578   __ verify_oop(r);
2579 }
2580 
2581 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2582 {
2583   const Register cache = r2;
2584   const Register index = r3;
2585   const Register obj   = r4;
2586   const Register off   = r19;
2587   const Register flags = r0;
2588   const Register raw_flags = r6;
2589   const Register bc    = r4; // uses same reg as obj, so don't mix them
2590 
2591   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2592   jvmti_post_field_access(cache, index, is_static, false);
2593   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2594 
2595   if (!is_static) {
2596     // obj is on the stack
2597     pop_and_check_object(obj);
2598   }
2599 
2600   // 8179954: We need to make sure that the code generated for
2601   // volatile accesses forms a sequentially-consistent set of
2602   // operations when combined with STLR and LDAR.  Without a leading
2603   // membar it's possible for a simple Dekker test to fail if loads
2604   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2605   // the stores in one method and we interpret the loads in another.
2606   if (! UseBarriersForVolatile) {
2607     Label notVolatile;
2608     __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2609     __ membar(MacroAssembler::AnyAny);
2610     __ bind(notVolatile);
2611   }
2612 
2613   const Address field(obj, off);
2614 
2615   Label Done, notByte, notBool, notInt, notShort, notChar,
2616               notLong, notFloat, notObj, notDouble;
2617 
2618   // x86 uses a shift and mask or wings it with a shift plus assert
2619   // the mask is not needed. aarch64 just uses bitfield extract
2620   __ ubfxw(flags, raw_flags, ConstantPoolCacheEntry::tos_state_shift, ConstantPoolCacheEntry::tos_state_bits);
2621 
2622   assert(btos == 0, "change code, btos != 0");
2623   __ cbnz(flags, notByte);
2624 
2625   // Don't rewrite getstatic, only getfield
2626   if (is_static) rc = may_not_rewrite;
2627 
2628   // btos
2629   __ access_load_at(T_BYTE, IN_HEAP, r0, field, noreg, noreg);
2630   __ push(btos);
2631   // Rewrite bytecode to be faster
2632   if (rc == may_rewrite) {
2633     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2634   }
2635   __ b(Done);
2636 
2637   __ bind(notByte);
2638   __ cmp(flags, (u1)ztos);
2639   __ br(Assembler::NE, notBool);
2640 
2641   // ztos (same code as btos)
2642   __ access_load_at(T_BOOLEAN, IN_HEAP, r0, field, noreg, noreg);
2643   __ push(ztos);
2644   // Rewrite bytecode to be faster
2645   if (rc == may_rewrite) {
2646     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2647     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2648   }
2649   __ b(Done);
2650 
2651   __ bind(notBool);
2652   __ cmp(flags, (u1)atos);
2653   __ br(Assembler::NE, notObj);
2654   // atos
2655   if (!EnableValhalla) {
2656     do_oop_load(_masm, field, r0, IN_HEAP);
2657     __ push(atos);
2658     if (rc == may_rewrite) {
2659       patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2660     }  
2661     __ b(Done);
2662   } else { // Valhalla
2663 
2664     if (is_static) {
2665       __ load_heap_oop(r0, field);
2666       Label isFlattenable, isUninitialized;
2667       // Issue below if the static field has not been initialized yet
2668       __ test_field_is_flattenable(raw_flags, r8 /*temp*/, isFlattenable);
2669         // Not flattenable case
2670         __ push(atos);
2671         __ b(Done);
2672       // Flattenable case, must not return null even if uninitialized
2673       __ bind(isFlattenable);
2674         __ cbz(r0, isUninitialized);
2675           __ push(atos);
2676           __ b(Done);
2677         __ bind(isUninitialized);
2678           __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2679           __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_value_field), obj, raw_flags);
2680           __ verify_oop(r0);
2681           __ push(atos);
2682           __ b(Done);
2683     } else {
2684       Label isFlattened, isInitialized, isFlattenable, rewriteFlattenable;
2685         __ test_field_is_flattenable(raw_flags, r8 /*temp*/, isFlattenable);
2686         // Non-flattenable field case, also covers the object case
2687         __ load_heap_oop(r0, field);
2688         __ push(atos);
2689         if (rc == may_rewrite) {
2690           patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2691         }
2692         __ b(Done);
2693       __ bind(isFlattenable);
2694         __ test_field_is_flattened(raw_flags, r8 /* temp */, isFlattened);
2695          // Non-flattened field case
2696           __ load_heap_oop(r0, field);
2697           __ cbnz(r0, isInitialized);
2698             __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2699             __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_instance_value_field), obj, raw_flags);
2700           __ bind(isInitialized);
2701           __ verify_oop(r0);
2702           __ push(atos);
2703           __ b(rewriteFlattenable);
2704         __ bind(isFlattened);
2705           __ ldr(r10, Address(cache, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset())));
2706           __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2707           call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), obj, raw_flags, r10); 
2708           __ verify_oop(r0);
2709           __ push(atos);
2710       __ bind(rewriteFlattenable);
2711       if (rc == may_rewrite) { 
2712          patch_bytecode(Bytecodes::_fast_qgetfield, bc, r1);
2713       }
2714       __ b(Done);
2715     }
2716   }
2717 
2718   __ bind(notObj);
2719   __ cmp(flags, (u1)itos);
2720   __ br(Assembler::NE, notInt);
2721   // itos
2722   __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
2723   __ push(itos);
2724   // Rewrite bytecode to be faster
2725   if (rc == may_rewrite) {
2726     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2727   }
2728   __ b(Done);
2729 
2730   __ bind(notInt);
2731   __ cmp(flags, (u1)ctos);
2732   __ br(Assembler::NE, notChar);
2733   // ctos
2734   __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
2735   __ push(ctos);
2736   // Rewrite bytecode to be faster
2737   if (rc == may_rewrite) {
2738     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2739   }
2740   __ b(Done);
2741 
2742   __ bind(notChar);
2743   __ cmp(flags, (u1)stos);
2744   __ br(Assembler::NE, notShort);
2745   // stos
2746   __ access_load_at(T_SHORT, IN_HEAP, r0, field, noreg, noreg);
2747   __ push(stos);
2748   // Rewrite bytecode to be faster
2749   if (rc == may_rewrite) {
2750     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2751   }
2752   __ b(Done);
2753 
2754   __ bind(notShort);
2755   __ cmp(flags, (u1)ltos);
2756   __ br(Assembler::NE, notLong);
2757   // ltos
2758   __ access_load_at(T_LONG, IN_HEAP, r0, field, noreg, noreg);
2759   __ push(ltos);
2760   // Rewrite bytecode to be faster
2761   if (rc == may_rewrite) {
2762     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2763   }
2764   __ b(Done);
2765 
2766   __ bind(notLong);
2767   __ cmp(flags, (u1)ftos);
2768   __ br(Assembler::NE, notFloat);
2769   // ftos
2770   __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2771   __ push(ftos);
2772   // Rewrite bytecode to be faster
2773   if (rc == may_rewrite) {
2774     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2775   }
2776   __ b(Done);
2777 
2778   __ bind(notFloat);
2779 #ifdef ASSERT
2780   __ cmp(flags, (u1)dtos);
2781   __ br(Assembler::NE, notDouble);
2782 #endif
2783   // dtos
2784   __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2785   __ push(dtos);
2786   // Rewrite bytecode to be faster
2787   if (rc == may_rewrite) {
2788     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2789   }
2790 #ifdef ASSERT
2791   __ b(Done);
2792 
2793   __ bind(notDouble);
2794   __ stop("Bad state");
2795 #endif
2796 
2797   __ bind(Done);
2798 
2799   Label notVolatile;
2800   __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2801   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2802   __ bind(notVolatile);
2803 }
2804 
2805 
2806 void TemplateTable::getfield(int byte_no)
2807 {
2808   getfield_or_static(byte_no, false);
2809 }
2810 
2811 void TemplateTable::nofast_getfield(int byte_no) {
2812   getfield_or_static(byte_no, false, may_not_rewrite);
2813 }
2814 
2815 void TemplateTable::getstatic(int byte_no)
2816 {
2817   getfield_or_static(byte_no, true);
2818 }
2819 
2820 // The registers cache and index expected to be set before call.
2821 // The function may destroy various registers, just not the cache and index registers.
2822 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2823   transition(vtos, vtos);
2824 
2825   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2826 
2827   if (JvmtiExport::can_post_field_modification()) {
2828     // Check to see if a field modification watch has been set before
2829     // we take the time to call into the VM.
2830     Label L1;
2831     assert_different_registers(cache, index, r0);
2832     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2833     __ ldrw(r0, Address(rscratch1));
2834     __ cbz(r0, L1);
2835 
2836     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2837 
2838     if (is_static) {
2839       // Life is simple.  Null out the object pointer.
2840       __ mov(c_rarg1, zr);
2841     } else {
2842       // Life is harder. The stack holds the value on top, followed by
2843       // the object.  We don't know the size of the value, though; it
2844       // could be one or two words depending on its type. As a result,
2845       // we must find the type to determine where the object is.
2846       __ ldrw(c_rarg3, Address(c_rarg2,
2847                                in_bytes(cp_base_offset +
2848                                         ConstantPoolCacheEntry::flags_offset())));
2849       __ lsr(c_rarg3, c_rarg3,
2850              ConstantPoolCacheEntry::tos_state_shift);
2851       ConstantPoolCacheEntry::verify_tos_state_shift();
2852       Label nope2, done, ok;
2853       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2854       __ cmpw(c_rarg3, ltos);
2855       __ br(Assembler::EQ, ok);
2856       __ cmpw(c_rarg3, dtos);
2857       __ br(Assembler::NE, nope2);
2858       __ bind(ok);
2859       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2860       __ bind(nope2);
2861     }
2862     // cache entry pointer
2863     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2864     // object (tos)
2865     __ mov(c_rarg3, esp);
2866     // c_rarg1: object pointer set up above (NULL if static)
2867     // c_rarg2: cache entry pointer
2868     // c_rarg3: jvalue object on the stack
2869     __ call_VM(noreg,
2870                CAST_FROM_FN_PTR(address,
2871                                 InterpreterRuntime::post_field_modification),
2872                c_rarg1, c_rarg2, c_rarg3);
2873     __ get_cache_and_index_at_bcp(cache, index, 1);
2874     __ bind(L1);
2875   }
2876 }
2877 
2878 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2879   transition(vtos, vtos);
2880 
2881   const Register cache = r2;
2882   const Register index = r3;
2883   const Register obj   = r2;
2884   const Register off   = r19;
2885   const Register flags = r0;
2886   const Register flags2 = r6;
2887   const Register bc    = r4;
2888 
2889   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2890   jvmti_post_field_mod(cache, index, is_static);
2891   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2892 
2893   Label Done;
2894   __ mov(r5, flags);
2895 
2896   {
2897     Label notVolatile;
2898     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2899     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2900     __ bind(notVolatile);
2901   }
2902 
2903   // field address
2904   const Address field(obj, off);
2905 
2906   Label notByte, notBool, notInt, notShort, notChar,
2907         notLong, notFloat, notObj, notDouble;
2908 
2909   __ mov(flags2, flags); 
2910 
2911   // x86 uses a shift and mask or wings it with a shift plus assert
2912   // the mask is not needed. aarch64 just uses bitfield extract
2913   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2914 
2915   assert(btos == 0, "change code, btos != 0");
2916   __ cbnz(flags, notByte);
2917 
2918   // Don't rewrite putstatic, only putfield
2919   if (is_static) rc = may_not_rewrite;
2920 
2921   // btos
2922   {
2923     __ pop(btos);
2924     if (!is_static) pop_and_check_object(obj);
2925     __ access_store_at(T_BYTE, IN_HEAP, field, r0, noreg, noreg);
2926     if (rc == may_rewrite) {
2927       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2928     }
2929     __ b(Done);
2930   }
2931 
2932   __ bind(notByte);
2933   __ cmp(flags, (u1)ztos);
2934   __ br(Assembler::NE, notBool);
2935 
2936   // ztos
2937   {
2938     __ pop(ztos);
2939     if (!is_static) pop_and_check_object(obj);
2940     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg);
2941     if (rc == may_rewrite) {
2942       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2943     }
2944     __ b(Done);
2945   }
2946 
2947   __ bind(notBool);
2948   __ cmp(flags, (u1)atos);
2949   __ br(Assembler::NE, notObj);
2950 
2951   // atos
2952   {
2953      if (!EnableValhalla) {
2954       __ pop(atos);
2955       if (!is_static) pop_and_check_object(obj);
2956       // Store into the field
2957       do_oop_store(_masm, field, r0, IN_HEAP);
2958       if (rc == may_rewrite) {
2959         patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2960       }
2961       __ b(Done);
2962      } else { // Valhalla
2963 
2964       __ pop(atos);
2965       if (is_static) {
2966         Label notFlattenable;
2967          __ test_field_is_not_flattenable(flags2, r8 /* temp */, notFlattenable);
2968          __ null_check(r0);
2969          __ bind(notFlattenable);
2970          do_oop_store(_masm, field, r0, IN_HEAP); 
2971          __ b(Done);
2972       } else {
2973         Label isFlattenable, isFlattened, notBuffered, notBuffered2, rewriteNotFlattenable, rewriteFlattenable;
2974         __ test_field_is_flattenable(flags2, r8 /*temp*/, isFlattenable);
2975         // Not flattenable case, covers not flattenable values and objects
2976         pop_and_check_object(obj);
2977         // Store into the field
2978         do_oop_store(_masm, field, r0, IN_HEAP);
2979         __ bind(rewriteNotFlattenable);
2980         if (rc == may_rewrite) {
2981           patch_bytecode(Bytecodes::_fast_aputfield, bc, r19, true, byte_no); 
2982         }
2983         __ b(Done);
2984         // Implementation of the flattenable semantic
2985         __ bind(isFlattenable);
2986         __ null_check(r0);
2987         __ test_field_is_flattened(flags2, r8 /*temp*/, isFlattened);
2988         // Not flattened case
2989         pop_and_check_object(obj);
2990         // Store into the field
2991         do_oop_store(_masm, field, r0, IN_HEAP);
2992         __ b(rewriteFlattenable);
2993         __ bind(isFlattened);
2994         pop_and_check_object(obj);
2995         call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flattened_value), r0, off, obj);
2996         __ bind(rewriteFlattenable);
2997         if (rc == may_rewrite) {
2998           patch_bytecode(Bytecodes::_fast_qputfield, bc, r19, true, byte_no);
2999         }
3000         __ b(Done);
3001       }
3002      }  // Valhalla
3003   }
3004 
3005   __ bind(notObj);
3006   __ cmp(flags, (u1)itos);
3007   __ br(Assembler::NE, notInt);
3008 
3009   // itos
3010   {
3011     __ pop(itos);
3012     if (!is_static) pop_and_check_object(obj);
3013     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg);
3014     if (rc == may_rewrite) {
3015       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
3016     }
3017     __ b(Done);
3018   }
3019 
3020   __ bind(notInt);
3021   __ cmp(flags, (u1)ctos);
3022   __ br(Assembler::NE, notChar);
3023 
3024   // ctos
3025   {
3026     __ pop(ctos);
3027     if (!is_static) pop_and_check_object(obj);
3028     __ access_store_at(T_CHAR, IN_HEAP, field, r0, noreg, noreg);
3029     if (rc == may_rewrite) {
3030       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
3031     }
3032     __ b(Done);
3033   }
3034 
3035   __ bind(notChar);
3036   __ cmp(flags, (u1)stos);
3037   __ br(Assembler::NE, notShort);
3038 
3039   // stos
3040   {
3041     __ pop(stos);
3042     if (!is_static) pop_and_check_object(obj);
3043     __ access_store_at(T_SHORT, IN_HEAP, field, r0, noreg, noreg);
3044     if (rc == may_rewrite) {
3045       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
3046     }
3047     __ b(Done);
3048   }
3049 
3050   __ bind(notShort);
3051   __ cmp(flags, (u1)ltos);
3052   __ br(Assembler::NE, notLong);
3053 
3054   // ltos
3055   {
3056     __ pop(ltos);
3057     if (!is_static) pop_and_check_object(obj);
3058     __ access_store_at(T_LONG, IN_HEAP, field, r0, noreg, noreg);
3059     if (rc == may_rewrite) {
3060       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
3061     }
3062     __ b(Done);
3063   }
3064 
3065   __ bind(notLong);
3066   __ cmp(flags, (u1)ftos);
3067   __ br(Assembler::NE, notFloat);
3068 
3069   // ftos
3070   {
3071     __ pop(ftos);
3072     if (!is_static) pop_and_check_object(obj);
3073     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
3074     if (rc == may_rewrite) {
3075       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
3076     }
3077     __ b(Done);
3078   }
3079 
3080   __ bind(notFloat);
3081 #ifdef ASSERT
3082   __ cmp(flags, (u1)dtos);
3083   __ br(Assembler::NE, notDouble);
3084 #endif
3085 
3086   // dtos
3087   {
3088     __ pop(dtos);
3089     if (!is_static) pop_and_check_object(obj);
3090     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
3091     if (rc == may_rewrite) {
3092       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
3093     }
3094   }
3095 
3096 #ifdef ASSERT
3097   __ b(Done);
3098 
3099   __ bind(notDouble);
3100   __ stop("Bad state");
3101 #endif
3102 
3103   __ bind(Done);
3104 
3105   {
3106     Label notVolatile;
3107     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3108     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
3109     __ bind(notVolatile);
3110   }
3111 }
3112 
3113 void TemplateTable::putfield(int byte_no)
3114 {
3115   putfield_or_static(byte_no, false);
3116 }
3117 
3118 void TemplateTable::nofast_putfield(int byte_no) {
3119   putfield_or_static(byte_no, false, may_not_rewrite);
3120 }
3121 
3122 void TemplateTable::putstatic(int byte_no) {
3123   putfield_or_static(byte_no, true);
3124 }
3125 
3126 void TemplateTable::jvmti_post_fast_field_mod()
3127 {
3128   if (JvmtiExport::can_post_field_modification()) {
3129     // Check to see if a field modification watch has been set before
3130     // we take the time to call into the VM.
3131     Label L2;
3132     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3133     __ ldrw(c_rarg3, Address(rscratch1));
3134     __ cbzw(c_rarg3, L2);
3135     __ pop_ptr(r19);                  // copy the object pointer from tos
3136     __ verify_oop(r19);
3137     __ push_ptr(r19);                 // put the object pointer back on tos
3138     // Save tos values before call_VM() clobbers them. Since we have
3139     // to do it for every data type, we use the saved values as the
3140     // jvalue object.
3141     switch (bytecode()) {          // load values into the jvalue object
3142     case Bytecodes::_fast_qputfield: //fall through
3143     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
3144     case Bytecodes::_fast_bputfield: // fall through
3145     case Bytecodes::_fast_zputfield: // fall through
3146     case Bytecodes::_fast_sputfield: // fall through
3147     case Bytecodes::_fast_cputfield: // fall through
3148     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
3149     case Bytecodes::_fast_dputfield: __ push_d(); break;
3150     case Bytecodes::_fast_fputfield: __ push_f(); break;
3151     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
3152 
3153     default:
3154       ShouldNotReachHere();
3155     }
3156     __ mov(c_rarg3, esp);             // points to jvalue on the stack
3157     // access constant pool cache entry
3158     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
3159     __ verify_oop(r19);
3160     // r19: object pointer copied above
3161     // c_rarg2: cache entry pointer
3162     // c_rarg3: jvalue object on the stack
3163     __ call_VM(noreg,
3164                CAST_FROM_FN_PTR(address,
3165                                 InterpreterRuntime::post_field_modification),
3166                r19, c_rarg2, c_rarg3);
3167 
3168     switch (bytecode()) {             // restore tos values
3169     case Bytecodes::_fast_qputfield: //fall through
3170     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
3171     case Bytecodes::_fast_bputfield: // fall through
3172     case Bytecodes::_fast_zputfield: // fall through
3173     case Bytecodes::_fast_sputfield: // fall through
3174     case Bytecodes::_fast_cputfield: // fall through
3175     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
3176     case Bytecodes::_fast_dputfield: __ pop_d(); break;
3177     case Bytecodes::_fast_fputfield: __ pop_f(); break;
3178     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
3179     default: break;
3180     }
3181     __ bind(L2);
3182   }
3183 }
3184 
3185 void TemplateTable::fast_storefield(TosState state)
3186 {
3187   transition(state, vtos);
3188 
3189   ByteSize base = ConstantPoolCache::base_offset();
3190 
3191   jvmti_post_fast_field_mod();
3192 
3193   // access constant pool cache
3194   __ get_cache_and_index_at_bcp(r2, r1, 1);
3195 
3196   // test for volatile with r3
3197   __ ldrw(r3, Address(r2, in_bytes(base +
3198                                    ConstantPoolCacheEntry::flags_offset())));
3199 
3200   // replace index with field offset from cache entry
3201   __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
3202 
3203   {
3204     Label notVolatile;
3205     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3206     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
3207     __ bind(notVolatile);
3208   }
3209 
3210   Label notVolatile;
3211 
3212   // Get object from stack
3213   pop_and_check_object(r2);
3214 
3215   // field address
3216   const Address field(r2, r1);
3217 
3218   // access field
3219   switch (bytecode()) {
3220   case Bytecodes::_fast_qputfield: //fall through 
3221    {
3222       Label isFlattened, done; 
3223       __ null_check(r0);
3224       __ test_field_is_flattened(r3, r8 /* temp */, isFlattened);
3225       // No Flattened case
3226       do_oop_store(_masm, field, r0, IN_HEAP);
3227       __ b(done);
3228       __ bind(isFlattened);
3229       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flattened_value), r0, r1, r2);
3230       __ bind(done);
3231     }
3232     break;
3233   case Bytecodes::_fast_aputfield:
3234     do_oop_store(_masm, field, r0, IN_HEAP);
3235     break;
3236   case Bytecodes::_fast_lputfield:
3237     __ access_store_at(T_LONG, IN_HEAP, field, r0, noreg, noreg);
3238     break;
3239   case Bytecodes::_fast_iputfield:
3240     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg);
3241     break;
3242   case Bytecodes::_fast_zputfield:
3243     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg);
3244     break;
3245   case Bytecodes::_fast_bputfield:
3246     __ access_store_at(T_BYTE, IN_HEAP, field, r0, noreg, noreg);
3247     break;
3248   case Bytecodes::_fast_sputfield:
3249     __ access_store_at(T_SHORT, IN_HEAP, field, r0, noreg, noreg);
3250     break;
3251   case Bytecodes::_fast_cputfield:
3252     __ access_store_at(T_CHAR, IN_HEAP, field, r0, noreg, noreg);
3253     break;
3254   case Bytecodes::_fast_fputfield:
3255     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
3256     break;
3257   case Bytecodes::_fast_dputfield:
3258     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
3259     break;
3260   default:
3261     ShouldNotReachHere();
3262   }
3263 
3264   {
3265     Label notVolatile;
3266     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3267     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
3268     __ bind(notVolatile);
3269   }
3270 }
3271 
3272 
3273 void TemplateTable::fast_accessfield(TosState state)
3274 {
3275   transition(atos, state);
3276   // Do the JVMTI work here to avoid disturbing the register state below
3277   if (JvmtiExport::can_post_field_access()) {
3278     // Check to see if a field access watch has been set before we
3279     // take the time to call into the VM.
3280     Label L1;
3281     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3282     __ ldrw(r2, Address(rscratch1));
3283     __ cbzw(r2, L1);
3284     // access constant pool cache entry
3285     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
3286     __ verify_oop(r0);
3287     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
3288     __ mov(c_rarg1, r0);
3289     // c_rarg1: object pointer copied above
3290     // c_rarg2: cache entry pointer
3291     __ call_VM(noreg,
3292                CAST_FROM_FN_PTR(address,
3293                                 InterpreterRuntime::post_field_access),
3294                c_rarg1, c_rarg2);
3295     __ pop_ptr(r0); // restore object pointer
3296     __ bind(L1);
3297   }
3298 
3299   // access constant pool cache
3300   __ get_cache_and_index_at_bcp(r2, r1, 1);
3301   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3302                                   ConstantPoolCacheEntry::f2_offset())));
3303   __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3304                                    ConstantPoolCacheEntry::flags_offset())));
3305 
3306   // r0: object
3307   __ verify_oop(r0);
3308   __ null_check(r0);
3309   const Address field(r0, r1);
3310 
3311   // 8179954: We need to make sure that the code generated for
3312   // volatile accesses forms a sequentially-consistent set of
3313   // operations when combined with STLR and LDAR.  Without a leading
3314   // membar it's possible for a simple Dekker test to fail if loads
3315   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3316   // the stores in one method and we interpret the loads in another.
3317   if (! UseBarriersForVolatile) {
3318     Label notVolatile;
3319     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3320     __ membar(MacroAssembler::AnyAny);
3321     __ bind(notVolatile);
3322   }
3323 
3324   // access field
3325   switch (bytecode()) {
3326   case Bytecodes::_fast_qgetfield: 
3327     {
3328        Label isFlattened, isInitialized, Done;
3329        // FIXME: We don't need to reload registers multiple times, but stay close to x86 code
3330        __ ldrw(r9, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()))); 
3331        __ test_field_is_flattened(r9, r8 /* temp */, isFlattened);
3332         // Non-flattened field case
3333         __ mov(r9, r0);
3334         __ load_heap_oop(r0, field);
3335         __ cbnz(r0, isInitialized);
3336           __ mov(r0, r9);
3337           __ ldrw(r9, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()))); 
3338           __ andw(r9, r9, ConstantPoolCacheEntry::field_index_mask);
3339           __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_instance_value_field), r0, r9);
3340         __ bind(isInitialized);
3341         __ verify_oop(r0);
3342         __ b(Done);
3343       __ bind(isFlattened);
3344         __ ldrw(r9, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset())));
3345         __ andw(r9, r9, ConstantPoolCacheEntry::field_index_mask);
3346         __ ldr(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset())));
3347         call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), r0, r9, r3);
3348         __ verify_oop(r0);
3349       __ bind(Done);
3350     }
3351     break;
3352   case Bytecodes::_fast_agetfield:
3353     do_oop_load(_masm, field, r0, IN_HEAP);
3354     __ verify_oop(r0);
3355     break;
3356   case Bytecodes::_fast_lgetfield:
3357     __ access_load_at(T_LONG, IN_HEAP, r0, field, noreg, noreg);
3358     break;
3359   case Bytecodes::_fast_igetfield:
3360     __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
3361     break;
3362   case Bytecodes::_fast_bgetfield:
3363     __ access_load_at(T_BYTE, IN_HEAP, r0, field, noreg, noreg);
3364     break;
3365   case Bytecodes::_fast_sgetfield:
3366     __ access_load_at(T_SHORT, IN_HEAP, r0, field, noreg, noreg);
3367     break;
3368   case Bytecodes::_fast_cgetfield:
3369     __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
3370     break;
3371   case Bytecodes::_fast_fgetfield:
3372     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3373     break;
3374   case Bytecodes::_fast_dgetfield:
3375     __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg);
3376     break;
3377   default:
3378     ShouldNotReachHere();
3379   }
3380   {
3381     Label notVolatile;
3382     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3383     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3384     __ bind(notVolatile);
3385   }
3386 }
3387 
3388 void TemplateTable::fast_xaccess(TosState state)
3389 {
3390   transition(vtos, state);
3391 
3392   // get receiver
3393   __ ldr(r0, aaddress(0));
3394   // access constant pool cache
3395   __ get_cache_and_index_at_bcp(r2, r3, 2);
3396   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3397                                   ConstantPoolCacheEntry::f2_offset())));
3398 
3399   // 8179954: We need to make sure that the code generated for
3400   // volatile accesses forms a sequentially-consistent set of
3401   // operations when combined with STLR and LDAR.  Without a leading
3402   // membar it's possible for a simple Dekker test to fail if loads
3403   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3404   // the stores in one method and we interpret the loads in another.
3405   if (! UseBarriersForVolatile) {
3406     Label notVolatile;
3407     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3408                                      ConstantPoolCacheEntry::flags_offset())));
3409     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3410     __ membar(MacroAssembler::AnyAny);
3411     __ bind(notVolatile);
3412   }
3413 
3414   // make sure exception is reported in correct bcp range (getfield is
3415   // next instruction)
3416   __ increment(rbcp);
3417   __ null_check(r0);
3418   switch (state) {
3419   case itos:
3420     __ access_load_at(T_INT, IN_HEAP, r0, Address(r0, r1, Address::lsl(0)), noreg, noreg);
3421     break;
3422   case atos:
3423     do_oop_load(_masm, Address(r0, r1, Address::lsl(0)), r0, IN_HEAP);
3424     __ verify_oop(r0);
3425     break;
3426   case ftos:
3427     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, Address(r0, r1, Address::lsl(0)), noreg, noreg);
3428     break;
3429   default:
3430     ShouldNotReachHere();
3431   }
3432 
3433   {
3434     Label notVolatile;
3435     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3436                                      ConstantPoolCacheEntry::flags_offset())));
3437     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3438     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3439     __ bind(notVolatile);
3440   }
3441 
3442   __ decrement(rbcp);
3443 }
3444 
3445 
3446 
3447 //-----------------------------------------------------------------------------
3448 // Calls
3449 
3450 void TemplateTable::count_calls(Register method, Register temp)
3451 {
3452   __ call_Unimplemented();
3453 }
3454 
3455 void TemplateTable::prepare_invoke(int byte_no,
3456                                    Register method, // linked method (or i-klass)
3457                                    Register index,  // itable index, MethodType, etc.
3458                                    Register recv,   // if caller wants to see it
3459                                    Register flags   // if caller wants to test it
3460                                    ) {
3461   // determine flags
3462   Bytecodes::Code code = bytecode();
3463   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3464   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3465   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3466   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3467   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3468   const bool load_receiver       = (recv  != noreg);
3469   const bool save_flags          = (flags != noreg);
3470   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3471   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3472   assert(flags == noreg || flags == r3, "");
3473   assert(recv  == noreg || recv  == r2, "");
3474 
3475   // setup registers & access constant pool cache
3476   if (recv  == noreg)  recv  = r2;
3477   if (flags == noreg)  flags = r3;
3478   assert_different_registers(method, index, recv, flags);
3479 
3480   // save 'interpreter return address'
3481   __ save_bcp();
3482 
3483   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3484 
3485   // maybe push appendix to arguments (just before return address)
3486   if (is_invokedynamic || is_invokehandle) {
3487     Label L_no_push;
3488     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3489     // Push the appendix as a trailing parameter.
3490     // This must be done before we get the receiver,
3491     // since the parameter_size includes it.
3492     __ push(r19);
3493     __ mov(r19, index);
3494     __ load_resolved_reference_at_index(index, r19);
3495     __ pop(r19);
3496     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3497     __ bind(L_no_push);
3498   }
3499 
3500   // load receiver if needed (note: no return address pushed yet)
3501   if (load_receiver) {
3502     __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3503     // FIXME -- is this actually correct? looks like it should be 2
3504     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3505     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3506     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3507     // __ movptr(recv, recv_addr);
3508     __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3509     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3510     __ verify_oop(recv);
3511   }
3512 
3513   // compute return type
3514   // x86 uses a shift and mask or wings it with a shift plus assert
3515   // the mask is not needed. aarch64 just uses bitfield extract
3516   __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3517   // load return address
3518   {
3519     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3520     __ mov(rscratch1, table_addr);
3521     __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3522   }
3523 }
3524 
3525 
3526 void TemplateTable::invokevirtual_helper(Register index,
3527                                          Register recv,
3528                                          Register flags)
3529 {
3530   // Uses temporary registers r0, r3
3531   assert_different_registers(index, recv, r0, r3);
3532   // Test for an invoke of a final method
3533   Label notFinal;
3534   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3535 
3536   const Register method = index;  // method must be rmethod
3537   assert(method == rmethod,
3538          "methodOop must be rmethod for interpreter calling convention");
3539 
3540   // do the call - the index is actually the method to call
3541   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3542 
3543   // It's final, need a null check here!
3544   __ null_check(recv);
3545 
3546   // profile this call
3547   __ profile_final_call(r0);
3548   __ profile_arguments_type(r0, method, r4, true);
3549 
3550   __ jump_from_interpreted(method, r0);
3551 
3552   __ bind(notFinal);
3553 
3554   // get receiver klass
3555   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3556   __ load_klass(r0, recv);
3557 
3558   // profile this call
3559   __ profile_virtual_call(r0, rlocals, r3);
3560 
3561   // get target methodOop & entry point
3562   __ lookup_virtual_method(r0, index, method);
3563   __ profile_arguments_type(r3, method, r4, true);
3564   // FIXME -- this looks completely redundant. is it?
3565   // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3566   __ jump_from_interpreted(method, r3);
3567 }
3568 
3569 void TemplateTable::invokevirtual(int byte_no)
3570 {
3571   transition(vtos, vtos);
3572   assert(byte_no == f2_byte, "use this argument");
3573 
3574   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3575 
3576   // rmethod: index (actually a Method*)
3577   // r2: receiver
3578   // r3: flags
3579 
3580   invokevirtual_helper(rmethod, r2, r3);
3581 }
3582 
3583 void TemplateTable::invokespecial(int byte_no)
3584 {
3585   transition(vtos, vtos);
3586   assert(byte_no == f1_byte, "use this argument");
3587 
3588   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3589                  r2);  // get receiver also for null check
3590   __ verify_oop(r2);
3591   __ null_check(r2);
3592   // do the call
3593   __ profile_call(r0);
3594   __ profile_arguments_type(r0, rmethod, rbcp, false);
3595   __ jump_from_interpreted(rmethod, r0);
3596 }
3597 
3598 void TemplateTable::invokestatic(int byte_no)
3599 {
3600   transition(vtos, vtos);
3601   assert(byte_no == f1_byte, "use this argument");
3602 
3603   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3604   // do the call
3605   __ profile_call(r0);
3606   __ profile_arguments_type(r0, rmethod, r4, false);
3607   __ jump_from_interpreted(rmethod, r0);
3608 }
3609 
3610 void TemplateTable::fast_invokevfinal(int byte_no)
3611 {
3612   __ call_Unimplemented();
3613 }
3614 
3615 void TemplateTable::invokeinterface(int byte_no) {
3616   transition(vtos, vtos);
3617   assert(byte_no == f1_byte, "use this argument");
3618 
3619   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 Method*
3620                  r2, r3); // recv, flags
3621 
3622   // r0: interface klass (from f1)
3623   // rmethod: method (from f2)
3624   // r2: receiver
3625   // r3: flags
3626 
3627   // First check for Object case, then private interface method,
3628   // then regular interface method.
3629 
3630   // Special case of invokeinterface called for virtual method of
3631   // java.lang.Object.  See cpCache.cpp for details.
3632   Label notObjectMethod;
3633   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notObjectMethod);
3634 
3635   invokevirtual_helper(rmethod, r2, r3);
3636   __ bind(notObjectMethod);
3637 
3638   Label no_such_interface;
3639 
3640   // Check for private method invocation - indicated by vfinal
3641   Label notVFinal;
3642   __ tbz(r3, ConstantPoolCacheEntry::is_vfinal_shift, notVFinal);
3643 
3644   // Get receiver klass into r3 - also a null check
3645   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3646   __ load_klass(r3, r2);
3647 
3648   Label subtype;
3649   __ check_klass_subtype(r3, r0, r4, subtype);
3650   // If we get here the typecheck failed
3651   __ b(no_such_interface);
3652   __ bind(subtype);
3653 
3654   __ profile_final_call(r0);
3655   __ profile_arguments_type(r0, rmethod, r4, true);
3656   __ jump_from_interpreted(rmethod, r0);
3657 
3658   __ bind(notVFinal);
3659 
3660   // Get receiver klass into r3 - also a null check
3661   __ restore_locals();
3662   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3663   __ load_klass(r3, r2);
3664 
3665   Label no_such_method;
3666 
3667   // Preserve method for throw_AbstractMethodErrorVerbose.
3668   __ mov(r16, rmethod);
3669   // Receiver subtype check against REFC.
3670   // Superklass in r0. Subklass in r3. Blows rscratch2, r13
3671   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3672                              r3, r0, noreg,
3673                              // outputs: scan temp. reg, scan temp. reg
3674                              rscratch2, r13,
3675                              no_such_interface,
3676                              /*return_method=*/false);
3677 
3678   // profile this call
3679   __ profile_virtual_call(r3, r13, r19);
3680 
3681   // Get declaring interface class from method, and itable index
3682   __ ldr(r0, Address(rmethod, Method::const_offset()));
3683   __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
3684   __ ldr(r0, Address(r0, ConstantPool::pool_holder_offset_in_bytes()));
3685   __ ldrw(rmethod, Address(rmethod, Method::itable_index_offset()));
3686   __ subw(rmethod, rmethod, Method::itable_index_max);
3687   __ negw(rmethod, rmethod);
3688 
3689   // Preserve recvKlass for throw_AbstractMethodErrorVerbose.
3690   __ mov(rlocals, r3);
3691   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3692                              rlocals, r0, rmethod,
3693                              // outputs: method, scan temp. reg
3694                              rmethod, r13,
3695                              no_such_interface);
3696 
3697   // rmethod,: methodOop to call
3698   // r2: receiver
3699   // Check for abstract method error
3700   // Note: This should be done more efficiently via a throw_abstract_method_error
3701   //       interpreter entry point and a conditional jump to it in case of a null
3702   //       method.
3703   __ cbz(rmethod, no_such_method);
3704 
3705   __ profile_arguments_type(r3, rmethod, r13, true);
3706 
3707   // do the call
3708   // r2: receiver
3709   // rmethod,: methodOop
3710   __ jump_from_interpreted(rmethod, r3);
3711   __ should_not_reach_here();
3712 
3713   // exception handling code follows...
3714   // note: must restore interpreter registers to canonical
3715   //       state for exception handling to work correctly!
3716 
3717   __ bind(no_such_method);
3718   // throw exception
3719   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3720   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3721   // Pass arguments for generating a verbose error message.
3722   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose), r3, r16);
3723   // the call_VM checks for exception, so we should never return here.
3724   __ should_not_reach_here();
3725 
3726   __ bind(no_such_interface);
3727   // throw exception
3728   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3729   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3730   // Pass arguments for generating a verbose error message.
3731   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3732                    InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose), r3, r0);
3733   // the call_VM checks for exception, so we should never return here.
3734   __ should_not_reach_here();
3735   return;
3736 }
3737 
3738 void TemplateTable::invokehandle(int byte_no) {
3739   transition(vtos, vtos);
3740   assert(byte_no == f1_byte, "use this argument");
3741 
3742   prepare_invoke(byte_no, rmethod, r0, r2);
3743   __ verify_method_ptr(r2);
3744   __ verify_oop(r2);
3745   __ null_check(r2);
3746 
3747   // FIXME: profile the LambdaForm also
3748 
3749   // r13 is safe to use here as a scratch reg because it is about to
3750   // be clobbered by jump_from_interpreted().
3751   __ profile_final_call(r13);
3752   __ profile_arguments_type(r13, rmethod, r4, true);
3753 
3754   __ jump_from_interpreted(rmethod, r0);
3755 }
3756 
3757 void TemplateTable::invokedynamic(int byte_no) {
3758   transition(vtos, vtos);
3759   assert(byte_no == f1_byte, "use this argument");
3760 
3761   prepare_invoke(byte_no, rmethod, r0);
3762 
3763   // r0: CallSite object (from cpool->resolved_references[])
3764   // rmethod: MH.linkToCallSite method (from f2)
3765 
3766   // Note:  r0_callsite is already pushed by prepare_invoke
3767 
3768   // %%% should make a type profile for any invokedynamic that takes a ref argument
3769   // profile this call
3770   __ profile_call(rbcp);
3771   __ profile_arguments_type(r3, rmethod, r13, false);
3772 
3773   __ verify_oop(r0);
3774 
3775   __ jump_from_interpreted(rmethod, r0);
3776 }
3777 
3778 
3779 //-----------------------------------------------------------------------------
3780 // Allocation
3781 
3782 void TemplateTable::_new() {
3783   transition(vtos, atos);
3784 
3785   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3786   Label slow_case;
3787   Label done;
3788   Label initialize_header;
3789   Label initialize_object; // including clearing the fields
3790 
3791   __ get_cpool_and_tags(r4, r0);
3792   // Make sure the class we're about to instantiate has been resolved.
3793   // This is done before loading InstanceKlass to be consistent with the order
3794   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3795   const int tags_offset = Array<u1>::base_offset_in_bytes();
3796   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3797   __ lea(rscratch1, Address(rscratch1, tags_offset));
3798   __ ldarb(rscratch1, rscratch1);
3799   __ cmp(rscratch1, (u1)JVM_CONSTANT_Class);
3800   __ br(Assembler::NE, slow_case);
3801 
3802   // get InstanceKlass
3803   __ load_resolved_klass_at_offset(r4, r3, r4, rscratch1);
3804 
3805   // make sure klass is initialized & doesn't have finalizer
3806   // make sure klass is fully initialized
3807   __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3808   __ cmp(rscratch1, (u1)InstanceKlass::fully_initialized);
3809   __ br(Assembler::NE, slow_case);
3810 
3811   // get instance_size in InstanceKlass (scaled to a count of bytes)
3812   __ ldrw(r3,
3813           Address(r4,
3814                   Klass::layout_helper_offset()));
3815   // test to see if it has a finalizer or is malformed in some way
3816   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3817 
3818   // Allocate the instance:
3819   //  If TLAB is enabled:
3820   //    Try to allocate in the TLAB.
3821   //    If fails, go to the slow path.
3822   //  Else If inline contiguous allocations are enabled:
3823   //    Try to allocate in eden.
3824   //    If fails due to heap end, go to slow path.
3825   //
3826   //  If TLAB is enabled OR inline contiguous is enabled:
3827   //    Initialize the allocation.
3828   //    Exit.
3829   //
3830   //  Go to slow path.
3831   const bool allow_shared_alloc =
3832     Universe::heap()->supports_inline_contig_alloc();
3833 
3834   if (UseTLAB) {
3835     __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case);
3836 
3837     if (ZeroTLAB) {
3838       // the fields have been already cleared
3839       __ b(initialize_header);
3840     } else {
3841       // initialize both the header and fields
3842       __ b(initialize_object);
3843     }
3844   } else {
3845     // Allocation in the shared Eden, if allowed.
3846     //
3847     // r3: instance size in bytes
3848     if (allow_shared_alloc) {
3849       __ eden_allocate(r0, r3, 0, r10, slow_case);
3850     }
3851   }
3852 
3853   // If UseTLAB or allow_shared_alloc are true, the object is created above and
3854   // there is an initialize need. Otherwise, skip and go to the slow path.
3855   if (UseTLAB || allow_shared_alloc) {
3856     // The object is initialized before the header.  If the object size is
3857     // zero, go directly to the header initialization.
3858     __ bind(initialize_object);
3859     __ sub(r3, r3, sizeof(oopDesc));
3860     __ cbz(r3, initialize_header);
3861 
3862     // Initialize object fields
3863     {
3864       __ add(r2, r0, sizeof(oopDesc));
3865       Label loop;
3866       __ bind(loop);
3867       __ str(zr, Address(__ post(r2, BytesPerLong)));
3868       __ sub(r3, r3, BytesPerLong);
3869       __ cbnz(r3, loop);
3870     }
3871 
3872     // initialize object header only.
3873     __ bind(initialize_header);
3874     if (UseBiasedLocking) {
3875       __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3876     } else {
3877       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3878     }
3879     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3880     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3881     __ store_klass(r0, r4);      // store klass last
3882 
3883     {
3884       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3885       // Trigger dtrace event for fastpath
3886       __ push(atos); // save the return value
3887       __ call_VM_leaf(
3888            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3889       __ pop(atos); // restore the return value
3890 
3891     }
3892     __ b(done);
3893   }
3894 
3895   // slow case
3896   __ bind(slow_case);
3897   __ get_constant_pool(c_rarg1);
3898   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3899   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3900   __ verify_oop(r0);
3901 
3902   // continue
3903   __ bind(done);
3904   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3905   __ membar(Assembler::StoreStore);
3906 }
3907 
3908 void TemplateTable::defaultvalue() {
3909   transition(vtos, atos);
3910   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3911   __ get_constant_pool(c_rarg1);
3912   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::defaultvalue),
3913           c_rarg1, c_rarg2);
3914   __ verify_oop(r0);
3915   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3916   __ membar(Assembler::StoreStore);
3917 }
3918 
3919 void TemplateTable::withfield() {
3920   transition(vtos, atos);
3921   resolve_cache_and_index(f2_byte, c_rarg1 /*cache*/, c_rarg2 /*index*/, sizeof(u2));
3922 
3923   // n.b. unlike x86 cache is now rcpool plus the indexed offset
3924   // so using rcpool to meet shared code expectations
3925  
3926   call_VM(r1, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), rcpool);
3927   __ verify_oop(r1);
3928   __ add(esp, esp, r0);
3929   __ mov(r0, r1);
3930 }
3931 
3932 void TemplateTable::newarray() {
3933   transition(itos, atos);
3934   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3935   __ mov(c_rarg2, r0);
3936   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3937           c_rarg1, c_rarg2);
3938   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3939   __ membar(Assembler::StoreStore);
3940 }
3941 
3942 void TemplateTable::anewarray() {
3943   transition(itos, atos);
3944   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3945   __ get_constant_pool(c_rarg1);
3946   __ mov(c_rarg3, r0);
3947   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3948           c_rarg1, c_rarg2, c_rarg3);
3949   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3950   __ membar(Assembler::StoreStore);
3951 }
3952 
3953 void TemplateTable::arraylength() {
3954   transition(atos, itos);
3955   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3956   __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3957 }
3958 
3959 void TemplateTable::checkcast()
3960 {
3961   transition(atos, atos);
3962   Label done, is_null, ok_is_subtype, quicked, resolved;
3963   __ cbz(r0, is_null);
3964 
3965   // Get cpool & tags index
3966   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3967   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3968   // See if bytecode has already been quicked
3969   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3970   __ lea(r1, Address(rscratch1, r19));
3971   __ ldarb(r1, r1);
3972   __ cmp(r1, (u1)JVM_CONSTANT_Class);
3973   __ br(Assembler::EQ, quicked);
3974 
3975   __ push(atos); // save receiver for result, and for GC
3976   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3977   // vm_result_2 has metadata result
3978   __ get_vm_result_2(r0, rthread);
3979   __ pop(r3); // restore receiver
3980   __ b(resolved);
3981 
3982   // Get superklass in r0 and subklass in r3
3983   __ bind(quicked);
3984   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3985   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1); // r0 = klass
3986 
3987   __ bind(resolved);
3988   __ load_klass(r19, r3);
3989 
3990   // Generate subtype check.  Blows r2, r5.  Object in r3.
3991   // Superklass in r0.  Subklass in r19.
3992   __ gen_subtype_check(r19, ok_is_subtype);
3993 
3994   // Come here on failure
3995   __ push(r3);
3996   // object is at TOS
3997   __ b(Interpreter::_throw_ClassCastException_entry);
3998 
3999   // Come here on success
4000   __ bind(ok_is_subtype);
4001   __ mov(r0, r3); // Restore object in r3
4002 
4003   __ b(done);
4004   __ bind(is_null);
4005 
4006   // Collect counts on whether this test sees NULLs a lot or not.
4007   if (ProfileInterpreter) {
4008     __ profile_null_seen(r2);
4009   }
4010 
4011   if (EnableValhalla) {
4012     // Get cpool & tags index
4013     __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
4014     __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
4015      // See if bytecode has already been quicked
4016     __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
4017     __ lea(r1, Address(rscratch1, r19));
4018     __ ldarb(r1, r1); 
4019     // See if CP entry is a Q-descriptor
4020     __ andr (r1, r1, JVM_CONSTANT_QDescBit);
4021     __ cmp(r1, (u1) JVM_CONSTANT_QDescBit);
4022     __ br(Assembler::NE, done);
4023     __ b(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
4024   }
4025 
4026   __ bind(done);
4027 }
4028 
4029 void TemplateTable::instanceof() {
4030   transition(atos, itos);
4031   Label done, is_null, ok_is_subtype, quicked, resolved;
4032   __ cbz(r0, is_null);
4033 
4034   // Get cpool & tags index
4035   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
4036   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
4037   // See if bytecode has already been quicked
4038   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
4039   __ lea(r1, Address(rscratch1, r19));
4040   __ ldarb(r1, r1);
4041   __ cmp(r1, (u1)JVM_CONSTANT_Class);
4042   __ br(Assembler::EQ, quicked);
4043 
4044   __ push(atos); // save receiver for result, and for GC
4045   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4046   // vm_result_2 has metadata result
4047   __ get_vm_result_2(r0, rthread);
4048   __ pop(r3); // restore receiver
4049   __ verify_oop(r3);
4050   __ load_klass(r3, r3);
4051   __ b(resolved);
4052 
4053   // Get superklass in r0 and subklass in r3
4054   __ bind(quicked);
4055   __ load_klass(r3, r0);
4056   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1);
4057 
4058   __ bind(resolved);
4059 
4060   // Generate subtype check.  Blows r2, r5
4061   // Superklass in r0.  Subklass in r3.
4062   __ gen_subtype_check(r3, ok_is_subtype);
4063 
4064   // Come here on failure
4065   __ mov(r0, 0);
4066   __ b(done);
4067   // Come here on success
4068   __ bind(ok_is_subtype);
4069   __ mov(r0, 1);
4070 
4071   // Collect counts on whether this test sees NULLs a lot or not.
4072   if (ProfileInterpreter) {
4073     __ b(done);
4074     __ bind(is_null);
4075     __ profile_null_seen(r2);
4076   } else {
4077     __ bind(is_null);   // same as 'done'
4078   }
4079   __ bind(done);
4080   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
4081   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
4082 }
4083 
4084 //-----------------------------------------------------------------------------
4085 // Breakpoints
4086 void TemplateTable::_breakpoint() {
4087   // Note: We get here even if we are single stepping..
4088   // jbug inists on setting breakpoints at every bytecode
4089   // even if we are in single step mode.
4090 
4091   transition(vtos, vtos);
4092 
4093   // get the unpatched byte code
4094   __ get_method(c_rarg1);
4095   __ call_VM(noreg,
4096              CAST_FROM_FN_PTR(address,
4097                               InterpreterRuntime::get_original_bytecode_at),
4098              c_rarg1, rbcp);
4099   __ mov(r19, r0);
4100 
4101   // post the breakpoint event
4102   __ call_VM(noreg,
4103              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
4104              rmethod, rbcp);
4105 
4106   // complete the execution of original bytecode
4107   __ mov(rscratch1, r19);
4108   __ dispatch_only_normal(vtos);
4109 }
4110 
4111 //-----------------------------------------------------------------------------
4112 // Exceptions
4113 
4114 void TemplateTable::athrow() {
4115   transition(atos, vtos);
4116   __ null_check(r0);
4117   __ b(Interpreter::throw_exception_entry());
4118 }
4119 
4120 //-----------------------------------------------------------------------------
4121 // Synchronization
4122 //
4123 // Note: monitorenter & exit are symmetric routines; which is reflected
4124 //       in the assembly code structure as well
4125 //
4126 // Stack layout:
4127 //
4128 // [expressions  ] <--- esp               = expression stack top
4129 // ..
4130 // [expressions  ]
4131 // [monitor entry] <--- monitor block top = expression stack bot
4132 // ..
4133 // [monitor entry]
4134 // [frame data   ] <--- monitor block bot
4135 // ...
4136 // [saved rbp    ] <--- rbp
4137 void TemplateTable::monitorenter()
4138 {
4139   transition(atos, vtos);
4140 
4141   // check for NULL object
4142   __ null_check(r0);
4143 
4144   __ resolve(IS_NOT_NULL, r0);
4145 
4146   const Address monitor_block_top(
4147         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4148   const Address monitor_block_bot(
4149         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4150   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4151 
4152   Label allocated;
4153 
4154   // initialize entry pointer
4155   __ mov(c_rarg1, zr); // points to free slot or NULL
4156 
4157   // find a free slot in the monitor block (result in c_rarg1)
4158   {
4159     Label entry, loop, exit;
4160     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
4161                                         // starting with top-most entry
4162     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4163 
4164     __ b(entry);
4165 
4166     __ bind(loop);
4167     // check if current entry is used
4168     // if not used then remember entry in c_rarg1
4169     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
4170     __ cmp(zr, rscratch1);
4171     __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
4172     // check if current entry is for same object
4173     __ cmp(r0, rscratch1);
4174     // if same object then stop searching
4175     __ br(Assembler::EQ, exit);
4176     // otherwise advance to next entry
4177     __ add(c_rarg3, c_rarg3, entry_size);
4178     __ bind(entry);
4179     // check if bottom reached
4180     __ cmp(c_rarg3, c_rarg2);
4181     // if not at bottom then check this entry
4182     __ br(Assembler::NE, loop);
4183     __ bind(exit);
4184   }
4185 
4186   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
4187                             // if found, continue with that on
4188 
4189   // allocate one if there's no free slot
4190   {
4191     Label entry, loop;
4192     // 1. compute new pointers            // rsp: old expression stack top
4193     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
4194     __ sub(esp, esp, entry_size);         // move expression stack top
4195     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
4196     __ mov(c_rarg3, esp);                 // set start value for copy loop
4197     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
4198 
4199     __ sub(sp, sp, entry_size);           // make room for the monitor
4200 
4201     __ b(entry);
4202     // 2. move expression stack contents
4203     __ bind(loop);
4204     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
4205                                                    // word from old location
4206     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
4207     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
4208     __ bind(entry);
4209     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
4210     __ br(Assembler::NE, loop);      // if not at bottom then
4211                                      // copy next word
4212   }
4213 
4214   // call run-time routine
4215   // c_rarg1: points to monitor entry
4216   __ bind(allocated);
4217 
4218   // Increment bcp to point to the next bytecode, so exception
4219   // handling for async. exceptions work correctly.
4220   // The object has already been poped from the stack, so the
4221   // expression stack looks correct.
4222   __ increment(rbcp);
4223 
4224   // store object
4225   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
4226   __ lock_object(c_rarg1);
4227 
4228   // check to make sure this monitor doesn't cause stack overflow after locking
4229   __ save_bcp();  // in case of exception
4230   __ generate_stack_overflow_check(0);
4231 
4232   // The bcp has already been incremented. Just need to dispatch to
4233   // next instruction.
4234   __ dispatch_next(vtos);
4235 }
4236 
4237 
4238 void TemplateTable::monitorexit()
4239 {
4240   transition(atos, vtos);
4241 
4242   // check for NULL object
4243   __ null_check(r0);
4244 
4245   __ resolve(IS_NOT_NULL, r0);
4246 
4247   const Address monitor_block_top(
4248         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4249   const Address monitor_block_bot(
4250         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4251   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4252 
4253   Label found;
4254 
4255   // find matching slot
4256   {
4257     Label entry, loop;
4258     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
4259                                         // starting with top-most entry
4260     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4261                                         // of monitor block
4262     __ b(entry);
4263 
4264     __ bind(loop);
4265     // check if current entry is for same object
4266     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
4267     __ cmp(r0, rscratch1);
4268     // if same object then stop searching
4269     __ br(Assembler::EQ, found);
4270     // otherwise advance to next entry
4271     __ add(c_rarg1, c_rarg1, entry_size);
4272     __ bind(entry);
4273     // check if bottom reached
4274     __ cmp(c_rarg1, c_rarg2);
4275     // if not at bottom then check this entry
4276     __ br(Assembler::NE, loop);
4277   }
4278 
4279   // error handling. Unlocking was not block-structured
4280   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4281                    InterpreterRuntime::throw_illegal_monitor_state_exception));
4282   __ should_not_reach_here();
4283 
4284   // call run-time routine
4285   __ bind(found);
4286   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
4287   __ unlock_object(c_rarg1);
4288   __ pop_ptr(r0); // discard object
4289 }
4290 
4291 
4292 // Wide instructions
4293 void TemplateTable::wide()
4294 {
4295   __ load_unsigned_byte(r19, at_bcp(1));
4296   __ mov(rscratch1, (address)Interpreter::_wentry_point);
4297   __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
4298   __ br(rscratch1);
4299 }
4300 
4301 
4302 // Multi arrays
4303 void TemplateTable::multianewarray() {
4304   transition(vtos, atos);
4305   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
4306   // last dim is on top of stack; we want address of first one:
4307   // first_addr = last_addr + (ndims - 1) * wordSize
4308   __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
4309   __ sub(c_rarg1, c_rarg1, wordSize);
4310   call_VM(r0,
4311           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
4312           c_rarg1);
4313   __ load_unsigned_byte(r1, at_bcp(3));
4314   __ lea(esp, Address(esp, r1, Address::uxtw(3)));
4315 }
--- EOF ---