rev 54670 : Port of valuetypes to aarch64

   1 /*
   2  * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/barrierSetAssembler.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/interp_masm.hpp"
  32 #include "interpreter/templateTable.hpp"
  33 #include "memory/universe.hpp"
  34 #include "oops/methodData.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/objArrayKlass.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/frame.inline.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "runtime/stubRoutines.hpp"
  42 #include "runtime/synchronizer.hpp"
  43 
  44 #define __ _masm->
  45 
  46 // Platform-dependent initialization
  47 
  48 void TemplateTable::pd_initialize() {
  49   // No aarch64 specific initialization
  50 }
  51 
  52 // Address computation: local variables
  53 
  54 static inline Address iaddress(int n) {
  55   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  56 }
  57 
  58 static inline Address laddress(int n) {
  59   return iaddress(n + 1);
  60 }
  61 
  62 static inline Address faddress(int n) {
  63   return iaddress(n);
  64 }
  65 
  66 static inline Address daddress(int n) {
  67   return laddress(n);
  68 }
  69 
  70 static inline Address aaddress(int n) {
  71   return iaddress(n);
  72 }
  73 
  74 static inline Address iaddress(Register r) {
  75   return Address(rlocals, r, Address::lsl(3));
  76 }
  77 
  78 static inline Address laddress(Register r, Register scratch,
  79                                InterpreterMacroAssembler* _masm) {
  80   __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
  81   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  82 }
  83 
  84 static inline Address faddress(Register r) {
  85   return iaddress(r);
  86 }
  87 
  88 static inline Address daddress(Register r, Register scratch,
  89                                InterpreterMacroAssembler* _masm) {
  90   return laddress(r, scratch, _masm);
  91 }
  92 
  93 static inline Address aaddress(Register r) {
  94   return iaddress(r);
  95 }
  96 
  97 static inline Address at_rsp() {
  98   return Address(esp, 0);
  99 }
 100 
 101 // At top of Java expression stack which may be different than esp().  It
 102 // isn't for category 1 objects.
 103 static inline Address at_tos   () {
 104   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 105 }
 106 
 107 static inline Address at_tos_p1() {
 108   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 109 }
 110 
 111 static inline Address at_tos_p2() {
 112   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 113 }
 114 
 115 static inline Address at_tos_p3() {
 116   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 117 }
 118 
 119 static inline Address at_tos_p4() {
 120   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 121 }
 122 
 123 static inline Address at_tos_p5() {
 124   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 125 }
 126 
 127 // Condition conversion
 128 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 129   switch (cc) {
 130   case TemplateTable::equal        : return Assembler::NE;
 131   case TemplateTable::not_equal    : return Assembler::EQ;
 132   case TemplateTable::less         : return Assembler::GE;
 133   case TemplateTable::less_equal   : return Assembler::GT;
 134   case TemplateTable::greater      : return Assembler::LE;
 135   case TemplateTable::greater_equal: return Assembler::LT;
 136   }
 137   ShouldNotReachHere();
 138   return Assembler::EQ;
 139 }
 140 
 141 
 142 // Miscelaneous helper routines
 143 // Store an oop (or NULL) at the Address described by obj.
 144 // If val == noreg this means store a NULL
 145 static void do_oop_store(InterpreterMacroAssembler* _masm,
 146                          Address dst,
 147                          Register val,
 148                          DecoratorSet decorators) {
 149   assert(val == noreg || val == r0, "parameter is just for looks");
 150   __ store_heap_oop(dst, val, r10, r1, decorators);
 151 }
 152 
 153 static void do_oop_load(InterpreterMacroAssembler* _masm,
 154                         Address src,
 155                         Register dst,
 156                         DecoratorSet decorators) {
 157   __ load_heap_oop(dst, src, r10, r1, decorators);
 158 }
 159 
 160 Address TemplateTable::at_bcp(int offset) {
 161   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 162   return Address(rbcp, offset);
 163 }
 164 
 165 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 166                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 167                                    int byte_no)
 168 {
 169   if (!RewriteBytecodes)  return;
 170   Label L_patch_done;
 171 
 172   switch (bc) {
 173   case Bytecodes::_fast_qputfield:
 174   case Bytecodes::_fast_aputfield:
 175   case Bytecodes::_fast_bputfield:
 176   case Bytecodes::_fast_zputfield:
 177   case Bytecodes::_fast_cputfield:
 178   case Bytecodes::_fast_dputfield:
 179   case Bytecodes::_fast_fputfield:
 180   case Bytecodes::_fast_iputfield:
 181   case Bytecodes::_fast_lputfield:
 182   case Bytecodes::_fast_sputfield:
 183     {
 184       // We skip bytecode quickening for putfield instructions when
 185       // the put_code written to the constant pool cache is zero.
 186       // This is required so that every execution of this instruction
 187       // calls out to InterpreterRuntime::resolve_get_put to do
 188       // additional, required work.
 189       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 190       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 191       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 192       __ movw(bc_reg, bc);
 193       __ cbzw(temp_reg, L_patch_done);  // don't patch
 194     }
 195     break;
 196   default:
 197     assert(byte_no == -1, "sanity");
 198     // the pair bytecodes have already done the load.
 199     if (load_bc_into_bc_reg) {
 200       __ movw(bc_reg, bc);
 201     }
 202   }
 203 
 204   if (JvmtiExport::can_post_breakpoint()) {
 205     Label L_fast_patch;
 206     // if a breakpoint is present we can't rewrite the stream directly
 207     __ load_unsigned_byte(temp_reg, at_bcp(0));
 208     __ cmpw(temp_reg, Bytecodes::_breakpoint);
 209     __ br(Assembler::NE, L_fast_patch);
 210     // Let breakpoint table handling rewrite to quicker bytecode
 211     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 212     __ b(L_patch_done);
 213     __ bind(L_fast_patch);
 214   }
 215 
 216 #ifdef ASSERT
 217   Label L_okay;
 218   __ load_unsigned_byte(temp_reg, at_bcp(0));
 219   __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
 220   __ br(Assembler::EQ, L_okay);
 221   __ cmpw(temp_reg, bc_reg);
 222   __ br(Assembler::EQ, L_okay);
 223   __ stop("patching the wrong bytecode");
 224   __ bind(L_okay);
 225 #endif
 226 
 227   // patch bytecode
 228   __ strb(bc_reg, at_bcp(0));
 229   __ bind(L_patch_done);
 230 }
 231 
 232 
 233 // Individual instructions
 234 
 235 void TemplateTable::nop() {
 236   transition(vtos, vtos);
 237   // nothing to do
 238 }
 239 
 240 void TemplateTable::shouldnotreachhere() {
 241   transition(vtos, vtos);
 242   __ stop("shouldnotreachhere bytecode");
 243 }
 244 
 245 void TemplateTable::aconst_null()
 246 {
 247   transition(vtos, atos);
 248   __ mov(r0, 0);
 249 }
 250 
 251 void TemplateTable::iconst(int value)
 252 {
 253   transition(vtos, itos);
 254   __ mov(r0, value);
 255 }
 256 
 257 void TemplateTable::lconst(int value)
 258 {
 259   __ mov(r0, value);
 260 }
 261 
 262 void TemplateTable::fconst(int value)
 263 {
 264   transition(vtos, ftos);
 265   switch (value) {
 266   case 0:
 267     __ fmovs(v0, zr);
 268     break;
 269   case 1:
 270     __ fmovs(v0, 1.0);
 271     break;
 272   case 2:
 273     __ fmovs(v0, 2.0);
 274     break;
 275   default:
 276     ShouldNotReachHere();
 277     break;
 278   }
 279 }
 280 
 281 void TemplateTable::dconst(int value)
 282 {
 283   transition(vtos, dtos);
 284   switch (value) {
 285   case 0:
 286     __ fmovd(v0, zr);
 287     break;
 288   case 1:
 289     __ fmovd(v0, 1.0);
 290     break;
 291   case 2:
 292     __ fmovd(v0, 2.0);
 293     break;
 294   default:
 295     ShouldNotReachHere();
 296     break;
 297   }
 298 }
 299 
 300 void TemplateTable::bipush()
 301 {
 302   transition(vtos, itos);
 303   __ load_signed_byte32(r0, at_bcp(1));
 304 }
 305 
 306 void TemplateTable::sipush()
 307 {
 308   transition(vtos, itos);
 309   __ load_unsigned_short(r0, at_bcp(1));
 310   __ revw(r0, r0);
 311   __ asrw(r0, r0, 16);
 312 }
 313 
 314 void TemplateTable::ldc(bool wide)
 315 {
 316   transition(vtos, vtos);
 317   Label call_ldc, notFloat, notClass, notInt, Done;
 318 
 319   if (wide) {
 320     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 321   } else {
 322     __ load_unsigned_byte(r1, at_bcp(1));
 323   }
 324   __ get_cpool_and_tags(r2, r0);
 325 
 326   const int base_offset = ConstantPool::header_size() * wordSize;
 327   const int tags_offset = Array<u1>::base_offset_in_bytes();
 328 
 329   // get type
 330   __ add(r3, r1, tags_offset);
 331   __ lea(r3, Address(r0, r3));
 332   __ ldarb(r3, r3);
 333 
 334   // unresolved class - get the resolved class
 335   __ cmp(r3, (u1)JVM_CONSTANT_UnresolvedClass);
 336   __ br(Assembler::EQ, call_ldc);
 337 
 338   // unresolved class in error state - call into runtime to throw the error
 339   // from the first resolution attempt
 340   __ cmp(r3, (u1)JVM_CONSTANT_UnresolvedClassInError);
 341   __ br(Assembler::EQ, call_ldc);
 342 
 343   // resolved class - need to call vm to get java mirror of the class
 344   __ cmp(r3, (u1)JVM_CONSTANT_Class);
 345   __ br(Assembler::NE, notClass);
 346 
 347   __ bind(call_ldc);
 348   __ mov(c_rarg1, wide);
 349   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 350   __ push_ptr(r0);
 351   __ verify_oop(r0);
 352   __ b(Done);
 353 
 354   __ bind(notClass);
 355   __ cmp(r3, (u1)JVM_CONSTANT_Float);
 356   __ br(Assembler::NE, notFloat);
 357   // ftos
 358   __ adds(r1, r2, r1, Assembler::LSL, 3);
 359   __ ldrs(v0, Address(r1, base_offset));
 360   __ push_f();
 361   __ b(Done);
 362 
 363   __ bind(notFloat);
 364 
 365   __ cmp(r3, (u1)JVM_CONSTANT_Integer);
 366   __ br(Assembler::NE, notInt);
 367 
 368   // itos
 369   __ adds(r1, r2, r1, Assembler::LSL, 3);
 370   __ ldrw(r0, Address(r1, base_offset));
 371   __ push_i(r0);
 372   __ b(Done);
 373 
 374   __ bind(notInt);
 375   condy_helper(Done);
 376 
 377   __ bind(Done);
 378 }
 379 
 380 // Fast path for caching oop constants.
 381 void TemplateTable::fast_aldc(bool wide)
 382 {
 383   transition(vtos, atos);
 384 
 385   Register result = r0;
 386   Register tmp = r1;
 387   Register rarg = r2;
 388 
 389   int index_size = wide ? sizeof(u2) : sizeof(u1);
 390 
 391   Label resolved;
 392 
 393   // We are resolved if the resolved reference cache entry contains a
 394   // non-null object (String, MethodType, etc.)
 395   assert_different_registers(result, tmp);
 396   __ get_cache_index_at_bcp(tmp, 1, index_size);
 397   __ load_resolved_reference_at_index(result, tmp);
 398   __ cbnz(result, resolved);
 399 
 400   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 401 
 402   // first time invocation - must resolve first
 403   __ mov(rarg, (int)bytecode());
 404   __ call_VM(result, entry, rarg);
 405 
 406   __ bind(resolved);
 407 
 408   { // Check for the null sentinel.
 409     // If we just called the VM, it already did the mapping for us,
 410     // but it's harmless to retry.
 411     Label notNull;
 412 
 413     // Stash null_sentinel address to get its value later
 414     __ movptr(rarg, (uintptr_t)Universe::the_null_sentinel_addr());
 415     __ ldr(tmp, Address(rarg));
 416     __ cmpoop(result, tmp);
 417     __ br(Assembler::NE, notNull);
 418     __ mov(result, 0);  // NULL object reference
 419     __ bind(notNull);
 420   }
 421 
 422   if (VerifyOops) {
 423     // Safe to call with 0 result
 424     __ verify_oop(result);
 425   }
 426 }
 427 
 428 void TemplateTable::ldc2_w()
 429 {
 430   transition(vtos, vtos);
 431   Label notDouble, notLong, Done;
 432   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 433 
 434   __ get_cpool_and_tags(r1, r2);
 435   const int base_offset = ConstantPool::header_size() * wordSize;
 436   const int tags_offset = Array<u1>::base_offset_in_bytes();
 437 
 438   // get type
 439   __ lea(r2, Address(r2, r0, Address::lsl(0)));
 440   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 441   __ cmpw(r2, (int)JVM_CONSTANT_Double);
 442   __ br(Assembler::NE, notDouble);
 443 
 444   // dtos
 445   __ lea (r2, Address(r1, r0, Address::lsl(3)));
 446   __ ldrd(v0, Address(r2, base_offset));
 447   __ push_d();
 448   __ b(Done);
 449 
 450   __ bind(notDouble);
 451   __ cmpw(r2, (int)JVM_CONSTANT_Long);
 452   __ br(Assembler::NE, notLong);
 453 
 454   // ltos
 455   __ lea(r0, Address(r1, r0, Address::lsl(3)));
 456   __ ldr(r0, Address(r0, base_offset));
 457   __ push_l();
 458   __ b(Done);
 459 
 460   __ bind(notLong);
 461   condy_helper(Done);
 462 
 463   __ bind(Done);
 464 }
 465 
 466 void TemplateTable::condy_helper(Label& Done)
 467 {
 468   Register obj = r0;
 469   Register rarg = r1;
 470   Register flags = r2;
 471   Register off = r3;
 472 
 473   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 474 
 475   __ mov(rarg, (int) bytecode());
 476   __ call_VM(obj, entry, rarg);
 477 
 478   __ get_vm_result_2(flags, rthread);
 479 
 480   // VMr = obj = base address to find primitive value to push
 481   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 482   __ mov(off, flags);
 483   __ andw(off, off, ConstantPoolCacheEntry::field_index_mask);
 484 
 485   const Address field(obj, off);
 486 
 487   // What sort of thing are we loading?
 488   // x86 uses a shift and mask or wings it with a shift plus assert
 489   // the mask is not needed. aarch64 just uses bitfield extract
 490   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,
 491            ConstantPoolCacheEntry::tos_state_bits);
 492 
 493   switch (bytecode()) {
 494     case Bytecodes::_ldc:
 495     case Bytecodes::_ldc_w:
 496       {
 497         // tos in (itos, ftos, stos, btos, ctos, ztos)
 498         Label notInt, notFloat, notShort, notByte, notChar, notBool;
 499         __ cmpw(flags, itos);
 500         __ br(Assembler::NE, notInt);
 501         // itos
 502         __ ldrw(r0, field);
 503         __ push(itos);
 504         __ b(Done);
 505 
 506         __ bind(notInt);
 507         __ cmpw(flags, ftos);
 508         __ br(Assembler::NE, notFloat);
 509         // ftos
 510         __ load_float(field);
 511         __ push(ftos);
 512         __ b(Done);
 513 
 514         __ bind(notFloat);
 515         __ cmpw(flags, stos);
 516         __ br(Assembler::NE, notShort);
 517         // stos
 518         __ load_signed_short(r0, field);
 519         __ push(stos);
 520         __ b(Done);
 521 
 522         __ bind(notShort);
 523         __ cmpw(flags, btos);
 524         __ br(Assembler::NE, notByte);
 525         // btos
 526         __ load_signed_byte(r0, field);
 527         __ push(btos);
 528         __ b(Done);
 529 
 530         __ bind(notByte);
 531         __ cmpw(flags, ctos);
 532         __ br(Assembler::NE, notChar);
 533         // ctos
 534         __ load_unsigned_short(r0, field);
 535         __ push(ctos);
 536         __ b(Done);
 537 
 538         __ bind(notChar);
 539         __ cmpw(flags, ztos);
 540         __ br(Assembler::NE, notBool);
 541         // ztos
 542         __ load_signed_byte(r0, field);
 543         __ push(ztos);
 544         __ b(Done);
 545 
 546         __ bind(notBool);
 547         break;
 548       }
 549 
 550     case Bytecodes::_ldc2_w:
 551       {
 552         Label notLong, notDouble;
 553         __ cmpw(flags, ltos);
 554         __ br(Assembler::NE, notLong);
 555         // ltos
 556         __ ldr(r0, field);
 557         __ push(ltos);
 558         __ b(Done);
 559 
 560         __ bind(notLong);
 561         __ cmpw(flags, dtos);
 562         __ br(Assembler::NE, notDouble);
 563         // dtos
 564         __ load_double(field);
 565         __ push(dtos);
 566         __ b(Done);
 567 
 568        __ bind(notDouble);
 569         break;
 570       }
 571 
 572     default:
 573       ShouldNotReachHere();
 574     }
 575 
 576     __ stop("bad ldc/condy");
 577 }
 578 
 579 void TemplateTable::locals_index(Register reg, int offset)
 580 {
 581   __ ldrb(reg, at_bcp(offset));
 582   __ neg(reg, reg);
 583 }
 584 
 585 void TemplateTable::iload() {
 586   iload_internal();
 587 }
 588 
 589 void TemplateTable::nofast_iload() {
 590   iload_internal(may_not_rewrite);
 591 }
 592 
 593 void TemplateTable::iload_internal(RewriteControl rc) {
 594   transition(vtos, itos);
 595   if (RewriteFrequentPairs && rc == may_rewrite) {
 596     Label rewrite, done;
 597     Register bc = r4;
 598 
 599     // get next bytecode
 600     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 601 
 602     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 603     // last two iloads in a pair.  Comparing against fast_iload means that
 604     // the next bytecode is neither an iload or a caload, and therefore
 605     // an iload pair.
 606     __ cmpw(r1, Bytecodes::_iload);
 607     __ br(Assembler::EQ, done);
 608 
 609     // if _fast_iload rewrite to _fast_iload2
 610     __ cmpw(r1, Bytecodes::_fast_iload);
 611     __ movw(bc, Bytecodes::_fast_iload2);
 612     __ br(Assembler::EQ, rewrite);
 613 
 614     // if _caload rewrite to _fast_icaload
 615     __ cmpw(r1, Bytecodes::_caload);
 616     __ movw(bc, Bytecodes::_fast_icaload);
 617     __ br(Assembler::EQ, rewrite);
 618 
 619     // else rewrite to _fast_iload
 620     __ movw(bc, Bytecodes::_fast_iload);
 621 
 622     // rewrite
 623     // bc: new bytecode
 624     __ bind(rewrite);
 625     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 626     __ bind(done);
 627 
 628   }
 629 
 630   // do iload, get the local value into tos
 631   locals_index(r1);
 632   __ ldr(r0, iaddress(r1));
 633 
 634 }
 635 
 636 void TemplateTable::fast_iload2()
 637 {
 638   transition(vtos, itos);
 639   locals_index(r1);
 640   __ ldr(r0, iaddress(r1));
 641   __ push(itos);
 642   locals_index(r1, 3);
 643   __ ldr(r0, iaddress(r1));
 644 }
 645 
 646 void TemplateTable::fast_iload()
 647 {
 648   transition(vtos, itos);
 649   locals_index(r1);
 650   __ ldr(r0, iaddress(r1));
 651 }
 652 
 653 void TemplateTable::lload()
 654 {
 655   transition(vtos, ltos);
 656   __ ldrb(r1, at_bcp(1));
 657   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 658   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 659 }
 660 
 661 void TemplateTable::fload()
 662 {
 663   transition(vtos, ftos);
 664   locals_index(r1);
 665   // n.b. we use ldrd here because this is a 64 bit slot
 666   // this is comparable to the iload case
 667   __ ldrd(v0, faddress(r1));
 668 }
 669 
 670 void TemplateTable::dload()
 671 {
 672   transition(vtos, dtos);
 673   __ ldrb(r1, at_bcp(1));
 674   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 675   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 676 }
 677 
 678 void TemplateTable::aload()
 679 {
 680   transition(vtos, atos);
 681   locals_index(r1);
 682   __ ldr(r0, iaddress(r1));
 683 }
 684 
 685 void TemplateTable::locals_index_wide(Register reg) {
 686   __ ldrh(reg, at_bcp(2));
 687   __ rev16w(reg, reg);
 688   __ neg(reg, reg);
 689 }
 690 
 691 void TemplateTable::wide_iload() {
 692   transition(vtos, itos);
 693   locals_index_wide(r1);
 694   __ ldr(r0, iaddress(r1));
 695 }
 696 
 697 void TemplateTable::wide_lload()
 698 {
 699   transition(vtos, ltos);
 700   __ ldrh(r1, at_bcp(2));
 701   __ rev16w(r1, r1);
 702   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 703   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 704 }
 705 
 706 void TemplateTable::wide_fload()
 707 {
 708   transition(vtos, ftos);
 709   locals_index_wide(r1);
 710   // n.b. we use ldrd here because this is a 64 bit slot
 711   // this is comparable to the iload case
 712   __ ldrd(v0, faddress(r1));
 713 }
 714 
 715 void TemplateTable::wide_dload()
 716 {
 717   transition(vtos, dtos);
 718   __ ldrh(r1, at_bcp(2));
 719   __ rev16w(r1, r1);
 720   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 721   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 722 }
 723 
 724 void TemplateTable::wide_aload()
 725 {
 726   transition(vtos, atos);
 727   locals_index_wide(r1);
 728   __ ldr(r0, aaddress(r1));
 729 }
 730 
 731 void TemplateTable::index_check(Register array, Register index)
 732 {
 733   // destroys r1, rscratch1
 734   // check array
 735   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 736   // sign extend index for use by indexed load
 737   // __ movl2ptr(index, index);
 738   // check index
 739   Register length = rscratch1;
 740   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 741   __ cmpw(index, length);
 742   if (index != r1) {
 743     // ??? convention: move aberrant index into r1 for exception message
 744     assert(r1 != array, "different registers");
 745     __ mov(r1, index);
 746   }
 747   Label ok;
 748   __ br(Assembler::LO, ok);
 749   // ??? convention: move array into r3 for exception message
 750    __ mov(r3, array);
 751    __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 752    __ br(rscratch1);
 753   __ bind(ok);
 754 }
 755 
 756 void TemplateTable::iaload()
 757 {
 758   transition(itos, itos);
 759   __ mov(r1, r0);
 760   __ pop_ptr(r0);
 761   // r0: array
 762   // r1: index
 763   index_check(r0, r1); // leaves index in r1, kills rscratch1
 764   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
 765   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(2)), noreg, noreg);
 766 }
 767 
 768 void TemplateTable::laload()
 769 {
 770   transition(itos, ltos);
 771   __ mov(r1, r0);
 772   __ pop_ptr(r0);
 773   // r0: array
 774   // r1: index
 775   index_check(r0, r1); // leaves index in r1, kills rscratch1
 776   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
 777   __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(3)), noreg, noreg);
 778 }
 779 
 780 void TemplateTable::faload()
 781 {
 782   transition(itos, ftos);
 783   __ mov(r1, r0);
 784   __ pop_ptr(r0);
 785   // r0: array
 786   // r1: index
 787   index_check(r0, r1); // leaves index in r1, kills rscratch1
 788   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
 789   __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(2)), noreg, noreg);
 790 }
 791 
 792 void TemplateTable::daload()
 793 {
 794   transition(itos, dtos);
 795   __ mov(r1, r0);
 796   __ pop_ptr(r0);
 797   // r0: array
 798   // r1: index
 799   index_check(r0, r1); // leaves index in r1, kills rscratch1
 800   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
 801   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(3)), noreg, noreg);
 802 }
 803 
 804 void TemplateTable::aaload()
 805 {
 806   transition(itos, atos);
 807   __ mov(r1, r0);
 808   __ pop_ptr(r0);
 809   // r0: array
 810   // r1: index
 811   index_check(r0, r1); // leaves index in r1, kills rscratch1
 812   if (EnableValhalla && ValueArrayFlatten) {
 813     Label is_flat_array, done;
 814 
 815     __ test_flat_array_oop(r0, r10 /*temp*/, is_flat_array); 
 816     __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 817     do_oop_load(_masm, Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)), r0, IS_ARRAY);
 818 
 819     __ b(done);
 820     __ bind(is_flat_array);
 821     __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), r0, r1);
 822     __ bind(done);
 823   } else {
 824     __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 825     do_oop_load(_masm, Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)), r0, IS_ARRAY);
 826   }


 827 }
 828 
 829 void TemplateTable::baload()
 830 {
 831   transition(itos, itos);
 832   __ mov(r1, r0);
 833   __ pop_ptr(r0);
 834   // r0: array
 835   // r1: index
 836   index_check(r0, r1); // leaves index in r1, kills rscratch1
 837   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
 838   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(0)), noreg, noreg);
 839 }
 840 
 841 void TemplateTable::caload()
 842 {
 843   transition(itos, itos);
 844   __ mov(r1, r0);
 845   __ pop_ptr(r0);
 846   // r0: array
 847   // r1: index
 848   index_check(r0, r1); // leaves index in r1, kills rscratch1
 849   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 850   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 851 }
 852 
 853 // iload followed by caload frequent pair
 854 void TemplateTable::fast_icaload()
 855 {
 856   transition(vtos, itos);
 857   // load index out of locals
 858   locals_index(r2);
 859   __ ldr(r1, iaddress(r2));
 860 
 861   __ pop_ptr(r0);
 862 
 863   // r0: array
 864   // r1: index
 865   index_check(r0, r1); // leaves index in r1, kills rscratch1
 866   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 867   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 868 }
 869 
 870 void TemplateTable::saload()
 871 {
 872   transition(itos, itos);
 873   __ mov(r1, r0);
 874   __ pop_ptr(r0);
 875   // r0: array
 876   // r1: index
 877   index_check(r0, r1); // leaves index in r1, kills rscratch1
 878   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_SHORT) >> 1);
 879   __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(1)), noreg, noreg);
 880 }
 881 
 882 void TemplateTable::iload(int n)
 883 {
 884   transition(vtos, itos);
 885   __ ldr(r0, iaddress(n));
 886 }
 887 
 888 void TemplateTable::lload(int n)
 889 {
 890   transition(vtos, ltos);
 891   __ ldr(r0, laddress(n));
 892 }
 893 
 894 void TemplateTable::fload(int n)
 895 {
 896   transition(vtos, ftos);
 897   __ ldrs(v0, faddress(n));
 898 }
 899 
 900 void TemplateTable::dload(int n)
 901 {
 902   transition(vtos, dtos);
 903   __ ldrd(v0, daddress(n));
 904 }
 905 
 906 void TemplateTable::aload(int n)
 907 {
 908   transition(vtos, atos);
 909   __ ldr(r0, iaddress(n));
 910 }
 911 
 912 void TemplateTable::aload_0() {
 913   aload_0_internal();
 914 }
 915 
 916 void TemplateTable::nofast_aload_0() {
 917   aload_0_internal(may_not_rewrite);
 918 }
 919 
 920 void TemplateTable::aload_0_internal(RewriteControl rc) {
 921   // According to bytecode histograms, the pairs:
 922   //
 923   // _aload_0, _fast_igetfield
 924   // _aload_0, _fast_agetfield
 925   // _aload_0, _fast_fgetfield
 926   //
 927   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 928   // _aload_0 bytecode checks if the next bytecode is either
 929   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 930   // rewrites the current bytecode into a pair bytecode; otherwise it
 931   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 932   // the pair check anymore.
 933   //
 934   // Note: If the next bytecode is _getfield, the rewrite must be
 935   //       delayed, otherwise we may miss an opportunity for a pair.
 936   //
 937   // Also rewrite frequent pairs
 938   //   aload_0, aload_1
 939   //   aload_0, iload_1
 940   // These bytecodes with a small amount of code are most profitable
 941   // to rewrite
 942   if (RewriteFrequentPairs && rc == may_rewrite) {
 943     Label rewrite, done;
 944     const Register bc = r4;
 945 
 946     // get next bytecode
 947     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 948 
 949     // if _getfield then wait with rewrite
 950     __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
 951     __ br(Assembler::EQ, done);
 952 
 953     // if _igetfield then rewrite to _fast_iaccess_0
 954     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 955     __ cmpw(r1, Bytecodes::_fast_igetfield);
 956     __ movw(bc, Bytecodes::_fast_iaccess_0);
 957     __ br(Assembler::EQ, rewrite);
 958 
 959     // if _agetfield then rewrite to _fast_aaccess_0
 960     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 961     __ cmpw(r1, Bytecodes::_fast_agetfield);
 962     __ movw(bc, Bytecodes::_fast_aaccess_0);
 963     __ br(Assembler::EQ, rewrite);
 964 
 965     // if _fgetfield then rewrite to _fast_faccess_0
 966     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 967     __ cmpw(r1, Bytecodes::_fast_fgetfield);
 968     __ movw(bc, Bytecodes::_fast_faccess_0);
 969     __ br(Assembler::EQ, rewrite);
 970 
 971     // else rewrite to _fast_aload0
 972     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 973     __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
 974 
 975     // rewrite
 976     // bc: new bytecode
 977     __ bind(rewrite);
 978     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 979 
 980     __ bind(done);
 981   }
 982 
 983   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 984   aload(0);
 985 }
 986 
 987 void TemplateTable::istore()
 988 {
 989   transition(itos, vtos);
 990   locals_index(r1);
 991   // FIXME: We're being very pernickerty here storing a jint in a
 992   // local with strw, which costs an extra instruction over what we'd
 993   // be able to do with a simple str.  We should just store the whole
 994   // word.
 995   __ lea(rscratch1, iaddress(r1));
 996   __ strw(r0, Address(rscratch1));
 997 }
 998 
 999 void TemplateTable::lstore()
1000 {
1001   transition(ltos, vtos);
1002   locals_index(r1);
1003   __ str(r0, laddress(r1, rscratch1, _masm));
1004 }
1005 
1006 void TemplateTable::fstore() {
1007   transition(ftos, vtos);
1008   locals_index(r1);
1009   __ lea(rscratch1, iaddress(r1));
1010   __ strs(v0, Address(rscratch1));
1011 }
1012 
1013 void TemplateTable::dstore() {
1014   transition(dtos, vtos);
1015   locals_index(r1);
1016   __ strd(v0, daddress(r1, rscratch1, _masm));
1017 }
1018 
1019 void TemplateTable::astore()
1020 {
1021   transition(vtos, vtos);
1022   __ pop_ptr(r0);
1023   locals_index(r1);
1024   __ str(r0, aaddress(r1));
1025 }
1026 
1027 void TemplateTable::wide_istore() {
1028   transition(vtos, vtos);
1029   __ pop_i();
1030   locals_index_wide(r1);
1031   __ lea(rscratch1, iaddress(r1));
1032   __ strw(r0, Address(rscratch1));
1033 }
1034 
1035 void TemplateTable::wide_lstore() {
1036   transition(vtos, vtos);
1037   __ pop_l();
1038   locals_index_wide(r1);
1039   __ str(r0, laddress(r1, rscratch1, _masm));
1040 }
1041 
1042 void TemplateTable::wide_fstore() {
1043   transition(vtos, vtos);
1044   __ pop_f();
1045   locals_index_wide(r1);
1046   __ lea(rscratch1, faddress(r1));
1047   __ strs(v0, rscratch1);
1048 }
1049 
1050 void TemplateTable::wide_dstore() {
1051   transition(vtos, vtos);
1052   __ pop_d();
1053   locals_index_wide(r1);
1054   __ strd(v0, daddress(r1, rscratch1, _masm));
1055 }
1056 
1057 void TemplateTable::wide_astore() {
1058   transition(vtos, vtos);
1059   __ pop_ptr(r0);
1060   locals_index_wide(r1);
1061   __ str(r0, aaddress(r1));
1062 }
1063 
1064 void TemplateTable::iastore() {
1065   transition(itos, vtos);
1066   __ pop_i(r1);
1067   __ pop_ptr(r3);
1068   // r0: value
1069   // r1: index
1070   // r3: array
1071   index_check(r3, r1); // prefer index in r1
1072   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
1073   __ access_store_at(T_INT, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(2)), r0, noreg, noreg);
1074 }
1075 
1076 void TemplateTable::lastore() {
1077   transition(ltos, vtos);
1078   __ pop_i(r1);
1079   __ pop_ptr(r3);
1080   // r0: value
1081   // r1: index
1082   // r3: array
1083   index_check(r3, r1); // prefer index in r1
1084   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
1085   __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(3)), r0, noreg, noreg);
1086 }
1087 
1088 void TemplateTable::fastore() {
1089   transition(ftos, vtos);
1090   __ pop_i(r1);
1091   __ pop_ptr(r3);
1092   // v0: value
1093   // r1:  index
1094   // r3:  array
1095   index_check(r3, r1); // prefer index in r1
1096   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
1097   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(2)), noreg /* ftos */, noreg, noreg);
1098 }
1099 
1100 void TemplateTable::dastore() {
1101   transition(dtos, vtos);
1102   __ pop_i(r1);
1103   __ pop_ptr(r3);
1104   // v0: value
1105   // r1:  index
1106   // r3:  array
1107   index_check(r3, r1); // prefer index in r1
1108   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
1109   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(3)), noreg /* dtos */, noreg, noreg);
1110 }
1111 
1112 void TemplateTable::aastore() {
1113   Label is_null, ok_is_subtype, done;
1114   transition(vtos, vtos);
1115   // stack: ..., array, index, value
1116   __ ldr(r0, at_tos());    // value
1117   __ ldr(r2, at_tos_p1()); // index
1118   __ ldr(r3, at_tos_p2()); // array
1119 
1120   Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop));
1121 
1122   index_check(r3, r2);     // kills r1
1123 
1124   // DMS CHECK: what does line below do?
1125   __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); 
1126 
1127   // do array store check - check for NULL value first
1128   __ cbz(r0, is_null);
1129 
1130   // Load array klass to r1, check if it is flat and bail out to ususal way
1131   Label  is_flat_array;
1132   if (ValueArrayFlatten) {
1133     __ load_klass(r1, r3); 
1134     __ test_flat_array_klass(r1, r10 /*temp*/, is_flat_array);
1135   }
1136 
1137   // Move subklass into r1
1138   __ load_klass(r1, r0);
1139   // Move superklass into r0
1140   __ load_klass(r0, r3);
1141   __ ldr(r0, Address(r0,
1142          ObjArrayKlass::element_klass_offset()));
1143   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1144 
1145   // Generate subtype check.  Blows r2, r5
1146   // Superklass in r0.  Subklass in r1.
1147   __ gen_subtype_check(r1, ok_is_subtype);
1148 
1149   // Come here on failure
1150   // object is at TOS
1151   __ b(Interpreter::_throw_ArrayStoreException_entry);
1152 
1153   // Come here on success
1154   __ bind(ok_is_subtype);
1155 
1156   // Get the value we will store
1157   __ ldr(r0, at_tos());
1158   // Now store using the appropriate barrier
1159   do_oop_store(_masm, element_address, r0, IS_ARRAY);
1160   __ b(done);
1161 
1162   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1163   __ bind(is_null);
1164   __ profile_null_seen(r2);
1165 
1166   if (EnableValhalla) {
1167     Label is_null_into_value_array_npe, store_null;
1168 
1169     __ load_klass(r0, r3);
1170     // No way to store null in flat array
1171     __ test_flat_array_klass(r0, r1, is_null_into_value_array_npe);
1172 
1173     // Use case for storing values in objArray where element_klass is specifically
1174     // a value type because they could not be flattened "for reasons",
1175     // these need to have the same semantics as flat arrays, i.e. NPE
1176     __ ldr(r0, Address(r0, ObjArrayKlass::element_klass_offset()));
1177     __ test_klass_is_value(r0, r1, is_null_into_value_array_npe);
1178     __ b(store_null);
1179 
1180     __ bind(is_null_into_value_array_npe);
1181     __ b(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
1182 
1183     __ bind(store_null);
1184   }
1185 
1186   // Store a NULL
1187   do_oop_store(_masm, element_address, noreg, IS_ARRAY);
1188   __ b(done);
1189 
1190 
1191   if (EnableValhalla) { 
1192     // r0 - value, r2 - index, r3 - array. r1 - loaded array klass
1193     // store non-null value
1194     __ bind(is_flat_array);
1195 
1196     // Simplistic type check...
1197      Label is_type_ok;
1198 
1199     // Profile the not-null value's klass.
1200     // Load value class 
1201     __ load_klass(r10, r0);
1202     __ profile_typecheck(r2, r1, r0); // blows r2, and r0
1203 
1204     // flat value array needs exact type match
1205     // is "r10 == r0" (value subclass == array element superclass)
1206   
1207     // Move element klass into r0
1208      __ ldr(r0, Address(r1, ArrayKlass::element_klass_offset())); 
1209      __ cmp(r0, r10);
1210      __ br(Assembler::EQ, is_type_ok);
1211 
1212      __ profile_typecheck_failed(r2);
1213      __ b(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
1214      __ bind(is_type_ok);
1215 
1216     // DMS CHECK: Reload from TOS to be safe, 
1217     // DMS CHECK: Because of profile_typecheck that blows r2 and r0. Should we really do it?
1218      __ ldr(r1, at_tos());  // value
1219      __ mov(r2, r3); // array
1220      __ ldr(r3, at_tos_p1()); // index
1221      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_store), r1, r2, r3);
1222   }
1223 
1224 
1225   // Pop stack arguments
1226   __ bind(done);
1227   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1228 }
1229 
1230 void TemplateTable::bastore()
1231 {
1232   transition(itos, vtos);
1233   __ pop_i(r1);
1234   __ pop_ptr(r3);
1235   // r0: value
1236   // r1: index
1237   // r3: array
1238   index_check(r3, r1); // prefer index in r1
1239 
1240   // Need to check whether array is boolean or byte
1241   // since both types share the bastore bytecode.
1242   __ load_klass(r2, r3);
1243   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1244   int diffbit_index = exact_log2(Klass::layout_helper_boolean_diffbit());
1245   Label L_skip;
1246   __ tbz(r2, diffbit_index, L_skip);
1247   __ andw(r0, r0, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1248   __ bind(L_skip);
1249 
1250   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
1251   __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(0)), r0, noreg, noreg);
1252 }
1253 
1254 void TemplateTable::castore()
1255 {
1256   transition(itos, vtos);
1257   __ pop_i(r1);
1258   __ pop_ptr(r3);
1259   // r0: value
1260   // r1: index
1261   // r3: array
1262   index_check(r3, r1); // prefer index in r1
1263   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
1264   __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(1)), r0, noreg, noreg);
1265 }
1266 
1267 void TemplateTable::sastore()
1268 {
1269   castore();
1270 }
1271 
1272 void TemplateTable::istore(int n)
1273 {
1274   transition(itos, vtos);
1275   __ str(r0, iaddress(n));
1276 }
1277 
1278 void TemplateTable::lstore(int n)
1279 {
1280   transition(ltos, vtos);
1281   __ str(r0, laddress(n));
1282 }
1283 
1284 void TemplateTable::fstore(int n)
1285 {
1286   transition(ftos, vtos);
1287   __ strs(v0, faddress(n));
1288 }
1289 
1290 void TemplateTable::dstore(int n)
1291 {
1292   transition(dtos, vtos);
1293   __ strd(v0, daddress(n));
1294 }
1295 
1296 void TemplateTable::astore(int n)
1297 {
1298   transition(vtos, vtos);
1299   __ pop_ptr(r0);
1300   __ str(r0, iaddress(n));
1301 }
1302 
1303 void TemplateTable::pop()
1304 {
1305   transition(vtos, vtos);
1306   __ add(esp, esp, Interpreter::stackElementSize);
1307 }
1308 
1309 void TemplateTable::pop2()
1310 {
1311   transition(vtos, vtos);
1312   __ add(esp, esp, 2 * Interpreter::stackElementSize);
1313 }
1314 
1315 void TemplateTable::dup()
1316 {
1317   transition(vtos, vtos);
1318   __ ldr(r0, Address(esp, 0));
1319   __ push(r0);
1320   // stack: ..., a, a
1321 }
1322 
1323 void TemplateTable::dup_x1()
1324 {
1325   transition(vtos, vtos);
1326   // stack: ..., a, b
1327   __ ldr(r0, at_tos());  // load b
1328   __ ldr(r2, at_tos_p1());  // load a
1329   __ str(r0, at_tos_p1());  // store b
1330   __ str(r2, at_tos());  // store a
1331   __ push(r0);                  // push b
1332   // stack: ..., b, a, b
1333 }
1334 
1335 void TemplateTable::dup_x2()
1336 {
1337   transition(vtos, vtos);
1338   // stack: ..., a, b, c
1339   __ ldr(r0, at_tos());  // load c
1340   __ ldr(r2, at_tos_p2());  // load a
1341   __ str(r0, at_tos_p2());  // store c in a
1342   __ push(r0);      // push c
1343   // stack: ..., c, b, c, c
1344   __ ldr(r0, at_tos_p2());  // load b
1345   __ str(r2, at_tos_p2());  // store a in b
1346   // stack: ..., c, a, c, c
1347   __ str(r0, at_tos_p1());  // store b in c
1348   // stack: ..., c, a, b, c
1349 }
1350 
1351 void TemplateTable::dup2()
1352 {
1353   transition(vtos, vtos);
1354   // stack: ..., a, b
1355   __ ldr(r0, at_tos_p1());  // load a
1356   __ push(r0);                  // push a
1357   __ ldr(r0, at_tos_p1());  // load b
1358   __ push(r0);                  // push b
1359   // stack: ..., a, b, a, b
1360 }
1361 
1362 void TemplateTable::dup2_x1()
1363 {
1364   transition(vtos, vtos);
1365   // stack: ..., a, b, c
1366   __ ldr(r2, at_tos());  // load c
1367   __ ldr(r0, at_tos_p1());  // load b
1368   __ push(r0);                  // push b
1369   __ push(r2);                  // push c
1370   // stack: ..., a, b, c, b, c
1371   __ str(r2, at_tos_p3());  // store c in b
1372   // stack: ..., a, c, c, b, c
1373   __ ldr(r2, at_tos_p4());  // load a
1374   __ str(r2, at_tos_p2());  // store a in 2nd c
1375   // stack: ..., a, c, a, b, c
1376   __ str(r0, at_tos_p4());  // store b in a
1377   // stack: ..., b, c, a, b, c
1378 }
1379 
1380 void TemplateTable::dup2_x2()
1381 {
1382   transition(vtos, vtos);
1383   // stack: ..., a, b, c, d
1384   __ ldr(r2, at_tos());  // load d
1385   __ ldr(r0, at_tos_p1());  // load c
1386   __ push(r0)            ;      // push c
1387   __ push(r2);                  // push d
1388   // stack: ..., a, b, c, d, c, d
1389   __ ldr(r0, at_tos_p4());  // load b
1390   __ str(r0, at_tos_p2());  // store b in d
1391   __ str(r2, at_tos_p4());  // store d in b
1392   // stack: ..., a, d, c, b, c, d
1393   __ ldr(r2, at_tos_p5());  // load a
1394   __ ldr(r0, at_tos_p3());  // load c
1395   __ str(r2, at_tos_p3());  // store a in c
1396   __ str(r0, at_tos_p5());  // store c in a
1397   // stack: ..., c, d, a, b, c, d
1398 }
1399 
1400 void TemplateTable::swap()
1401 {
1402   transition(vtos, vtos);
1403   // stack: ..., a, b
1404   __ ldr(r2, at_tos_p1());  // load a
1405   __ ldr(r0, at_tos());  // load b
1406   __ str(r2, at_tos());  // store a in b
1407   __ str(r0, at_tos_p1());  // store b in a
1408   // stack: ..., b, a
1409 }
1410 
1411 void TemplateTable::iop2(Operation op)
1412 {
1413   transition(itos, itos);
1414   // r0 <== r1 op r0
1415   __ pop_i(r1);
1416   switch (op) {
1417   case add  : __ addw(r0, r1, r0); break;
1418   case sub  : __ subw(r0, r1, r0); break;
1419   case mul  : __ mulw(r0, r1, r0); break;
1420   case _and : __ andw(r0, r1, r0); break;
1421   case _or  : __ orrw(r0, r1, r0); break;
1422   case _xor : __ eorw(r0, r1, r0); break;
1423   case shl  : __ lslvw(r0, r1, r0); break;
1424   case shr  : __ asrvw(r0, r1, r0); break;
1425   case ushr : __ lsrvw(r0, r1, r0);break;
1426   default   : ShouldNotReachHere();
1427   }
1428 }
1429 
1430 void TemplateTable::lop2(Operation op)
1431 {
1432   transition(ltos, ltos);
1433   // r0 <== r1 op r0
1434   __ pop_l(r1);
1435   switch (op) {
1436   case add  : __ add(r0, r1, r0); break;
1437   case sub  : __ sub(r0, r1, r0); break;
1438   case mul  : __ mul(r0, r1, r0); break;
1439   case _and : __ andr(r0, r1, r0); break;
1440   case _or  : __ orr(r0, r1, r0); break;
1441   case _xor : __ eor(r0, r1, r0); break;
1442   default   : ShouldNotReachHere();
1443   }
1444 }
1445 
1446 void TemplateTable::idiv()
1447 {
1448   transition(itos, itos);
1449   // explicitly check for div0
1450   Label no_div0;
1451   __ cbnzw(r0, no_div0);
1452   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1453   __ br(rscratch1);
1454   __ bind(no_div0);
1455   __ pop_i(r1);
1456   // r0 <== r1 idiv r0
1457   __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1458 }
1459 
1460 void TemplateTable::irem()
1461 {
1462   transition(itos, itos);
1463   // explicitly check for div0
1464   Label no_div0;
1465   __ cbnzw(r0, no_div0);
1466   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1467   __ br(rscratch1);
1468   __ bind(no_div0);
1469   __ pop_i(r1);
1470   // r0 <== r1 irem r0
1471   __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1472 }
1473 
1474 void TemplateTable::lmul()
1475 {
1476   transition(ltos, ltos);
1477   __ pop_l(r1);
1478   __ mul(r0, r0, r1);
1479 }
1480 
1481 void TemplateTable::ldiv()
1482 {
1483   transition(ltos, ltos);
1484   // explicitly check for div0
1485   Label no_div0;
1486   __ cbnz(r0, no_div0);
1487   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1488   __ br(rscratch1);
1489   __ bind(no_div0);
1490   __ pop_l(r1);
1491   // r0 <== r1 ldiv r0
1492   __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1493 }
1494 
1495 void TemplateTable::lrem()
1496 {
1497   transition(ltos, ltos);
1498   // explicitly check for div0
1499   Label no_div0;
1500   __ cbnz(r0, no_div0);
1501   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1502   __ br(rscratch1);
1503   __ bind(no_div0);
1504   __ pop_l(r1);
1505   // r0 <== r1 lrem r0
1506   __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1507 }
1508 
1509 void TemplateTable::lshl()
1510 {
1511   transition(itos, ltos);
1512   // shift count is in r0
1513   __ pop_l(r1);
1514   __ lslv(r0, r1, r0);
1515 }
1516 
1517 void TemplateTable::lshr()
1518 {
1519   transition(itos, ltos);
1520   // shift count is in r0
1521   __ pop_l(r1);
1522   __ asrv(r0, r1, r0);
1523 }
1524 
1525 void TemplateTable::lushr()
1526 {
1527   transition(itos, ltos);
1528   // shift count is in r0
1529   __ pop_l(r1);
1530   __ lsrv(r0, r1, r0);
1531 }
1532 
1533 void TemplateTable::fop2(Operation op)
1534 {
1535   transition(ftos, ftos);
1536   switch (op) {
1537   case add:
1538     // n.b. use ldrd because this is a 64 bit slot
1539     __ pop_f(v1);
1540     __ fadds(v0, v1, v0);
1541     break;
1542   case sub:
1543     __ pop_f(v1);
1544     __ fsubs(v0, v1, v0);
1545     break;
1546   case mul:
1547     __ pop_f(v1);
1548     __ fmuls(v0, v1, v0);
1549     break;
1550   case div:
1551     __ pop_f(v1);
1552     __ fdivs(v0, v1, v0);
1553     break;
1554   case rem:
1555     __ fmovs(v1, v0);
1556     __ pop_f(v0);
1557     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::frem),
1558                          0, 2, MacroAssembler::ret_type_float);
1559     break;
1560   default:
1561     ShouldNotReachHere();
1562     break;
1563   }
1564 }
1565 
1566 void TemplateTable::dop2(Operation op)
1567 {
1568   transition(dtos, dtos);
1569   switch (op) {
1570   case add:
1571     // n.b. use ldrd because this is a 64 bit slot
1572     __ pop_d(v1);
1573     __ faddd(v0, v1, v0);
1574     break;
1575   case sub:
1576     __ pop_d(v1);
1577     __ fsubd(v0, v1, v0);
1578     break;
1579   case mul:
1580     __ pop_d(v1);
1581     __ fmuld(v0, v1, v0);
1582     break;
1583   case div:
1584     __ pop_d(v1);
1585     __ fdivd(v0, v1, v0);
1586     break;
1587   case rem:
1588     __ fmovd(v1, v0);
1589     __ pop_d(v0);
1590     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::drem),
1591                          0, 2, MacroAssembler::ret_type_double);
1592     break;
1593   default:
1594     ShouldNotReachHere();
1595     break;
1596   }
1597 }
1598 
1599 void TemplateTable::ineg()
1600 {
1601   transition(itos, itos);
1602   __ negw(r0, r0);
1603 
1604 }
1605 
1606 void TemplateTable::lneg()
1607 {
1608   transition(ltos, ltos);
1609   __ neg(r0, r0);
1610 }
1611 
1612 void TemplateTable::fneg()
1613 {
1614   transition(ftos, ftos);
1615   __ fnegs(v0, v0);
1616 }
1617 
1618 void TemplateTable::dneg()
1619 {
1620   transition(dtos, dtos);
1621   __ fnegd(v0, v0);
1622 }
1623 
1624 void TemplateTable::iinc()
1625 {
1626   transition(vtos, vtos);
1627   __ load_signed_byte(r1, at_bcp(2)); // get constant
1628   locals_index(r2);
1629   __ ldr(r0, iaddress(r2));
1630   __ addw(r0, r0, r1);
1631   __ str(r0, iaddress(r2));
1632 }
1633 
1634 void TemplateTable::wide_iinc()
1635 {
1636   transition(vtos, vtos);
1637   // __ mov(r1, zr);
1638   __ ldrw(r1, at_bcp(2)); // get constant and index
1639   __ rev16(r1, r1);
1640   __ ubfx(r2, r1, 0, 16);
1641   __ neg(r2, r2);
1642   __ sbfx(r1, r1, 16, 16);
1643   __ ldr(r0, iaddress(r2));
1644   __ addw(r0, r0, r1);
1645   __ str(r0, iaddress(r2));
1646 }
1647 
1648 void TemplateTable::convert()
1649 {
1650   // Checking
1651 #ifdef ASSERT
1652   {
1653     TosState tos_in  = ilgl;
1654     TosState tos_out = ilgl;
1655     switch (bytecode()) {
1656     case Bytecodes::_i2l: // fall through
1657     case Bytecodes::_i2f: // fall through
1658     case Bytecodes::_i2d: // fall through
1659     case Bytecodes::_i2b: // fall through
1660     case Bytecodes::_i2c: // fall through
1661     case Bytecodes::_i2s: tos_in = itos; break;
1662     case Bytecodes::_l2i: // fall through
1663     case Bytecodes::_l2f: // fall through
1664     case Bytecodes::_l2d: tos_in = ltos; break;
1665     case Bytecodes::_f2i: // fall through
1666     case Bytecodes::_f2l: // fall through
1667     case Bytecodes::_f2d: tos_in = ftos; break;
1668     case Bytecodes::_d2i: // fall through
1669     case Bytecodes::_d2l: // fall through
1670     case Bytecodes::_d2f: tos_in = dtos; break;
1671     default             : ShouldNotReachHere();
1672     }
1673     switch (bytecode()) {
1674     case Bytecodes::_l2i: // fall through
1675     case Bytecodes::_f2i: // fall through
1676     case Bytecodes::_d2i: // fall through
1677     case Bytecodes::_i2b: // fall through
1678     case Bytecodes::_i2c: // fall through
1679     case Bytecodes::_i2s: tos_out = itos; break;
1680     case Bytecodes::_i2l: // fall through
1681     case Bytecodes::_f2l: // fall through
1682     case Bytecodes::_d2l: tos_out = ltos; break;
1683     case Bytecodes::_i2f: // fall through
1684     case Bytecodes::_l2f: // fall through
1685     case Bytecodes::_d2f: tos_out = ftos; break;
1686     case Bytecodes::_i2d: // fall through
1687     case Bytecodes::_l2d: // fall through
1688     case Bytecodes::_f2d: tos_out = dtos; break;
1689     default             : ShouldNotReachHere();
1690     }
1691     transition(tos_in, tos_out);
1692   }
1693 #endif // ASSERT
1694   // static const int64_t is_nan = 0x8000000000000000L;
1695 
1696   // Conversion
1697   switch (bytecode()) {
1698   case Bytecodes::_i2l:
1699     __ sxtw(r0, r0);
1700     break;
1701   case Bytecodes::_i2f:
1702     __ scvtfws(v0, r0);
1703     break;
1704   case Bytecodes::_i2d:
1705     __ scvtfwd(v0, r0);
1706     break;
1707   case Bytecodes::_i2b:
1708     __ sxtbw(r0, r0);
1709     break;
1710   case Bytecodes::_i2c:
1711     __ uxthw(r0, r0);
1712     break;
1713   case Bytecodes::_i2s:
1714     __ sxthw(r0, r0);
1715     break;
1716   case Bytecodes::_l2i:
1717     __ uxtw(r0, r0);
1718     break;
1719   case Bytecodes::_l2f:
1720     __ scvtfs(v0, r0);
1721     break;
1722   case Bytecodes::_l2d:
1723     __ scvtfd(v0, r0);
1724     break;
1725   case Bytecodes::_f2i:
1726   {
1727     Label L_Okay;
1728     __ clear_fpsr();
1729     __ fcvtzsw(r0, v0);
1730     __ get_fpsr(r1);
1731     __ cbzw(r1, L_Okay);
1732     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2i),
1733                          0, 1, MacroAssembler::ret_type_integral);
1734     __ bind(L_Okay);
1735   }
1736     break;
1737   case Bytecodes::_f2l:
1738   {
1739     Label L_Okay;
1740     __ clear_fpsr();
1741     __ fcvtzs(r0, v0);
1742     __ get_fpsr(r1);
1743     __ cbzw(r1, L_Okay);
1744     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2l),
1745                          0, 1, MacroAssembler::ret_type_integral);
1746     __ bind(L_Okay);
1747   }
1748     break;
1749   case Bytecodes::_f2d:
1750     __ fcvts(v0, v0);
1751     break;
1752   case Bytecodes::_d2i:
1753   {
1754     Label L_Okay;
1755     __ clear_fpsr();
1756     __ fcvtzdw(r0, v0);
1757     __ get_fpsr(r1);
1758     __ cbzw(r1, L_Okay);
1759     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2i),
1760                          0, 1, MacroAssembler::ret_type_integral);
1761     __ bind(L_Okay);
1762   }
1763     break;
1764   case Bytecodes::_d2l:
1765   {
1766     Label L_Okay;
1767     __ clear_fpsr();
1768     __ fcvtzd(r0, v0);
1769     __ get_fpsr(r1);
1770     __ cbzw(r1, L_Okay);
1771     __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2l),
1772                          0, 1, MacroAssembler::ret_type_integral);
1773     __ bind(L_Okay);
1774   }
1775     break;
1776   case Bytecodes::_d2f:
1777     __ fcvtd(v0, v0);
1778     break;
1779   default:
1780     ShouldNotReachHere();
1781   }
1782 }
1783 
1784 void TemplateTable::lcmp()
1785 {
1786   transition(ltos, itos);
1787   Label done;
1788   __ pop_l(r1);
1789   __ cmp(r1, r0);
1790   __ mov(r0, (u_int64_t)-1L);
1791   __ br(Assembler::LT, done);
1792   // __ mov(r0, 1UL);
1793   // __ csel(r0, r0, zr, Assembler::NE);
1794   // and here is a faster way
1795   __ csinc(r0, zr, zr, Assembler::EQ);
1796   __ bind(done);
1797 }
1798 
1799 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1800 {
1801   Label done;
1802   if (is_float) {
1803     // XXX get rid of pop here, use ... reg, mem32
1804     __ pop_f(v1);
1805     __ fcmps(v1, v0);
1806   } else {
1807     // XXX get rid of pop here, use ... reg, mem64
1808     __ pop_d(v1);
1809     __ fcmpd(v1, v0);
1810   }
1811   if (unordered_result < 0) {
1812     // we want -1 for unordered or less than, 0 for equal and 1 for
1813     // greater than.
1814     __ mov(r0, (u_int64_t)-1L);
1815     // for FP LT tests less than or unordered
1816     __ br(Assembler::LT, done);
1817     // install 0 for EQ otherwise 1
1818     __ csinc(r0, zr, zr, Assembler::EQ);
1819   } else {
1820     // we want -1 for less than, 0 for equal and 1 for unordered or
1821     // greater than.
1822     __ mov(r0, 1L);
1823     // for FP HI tests greater than or unordered
1824     __ br(Assembler::HI, done);
1825     // install 0 for EQ otherwise ~0
1826     __ csinv(r0, zr, zr, Assembler::EQ);
1827 
1828   }
1829   __ bind(done);
1830 }
1831 
1832 void TemplateTable::branch(bool is_jsr, bool is_wide)
1833 {
1834   // We might be moving to a safepoint.  The thread which calls
1835   // Interpreter::notice_safepoints() will effectively flush its cache
1836   // when it makes a system call, but we need to do something to
1837   // ensure that we see the changed dispatch table.
1838   __ membar(MacroAssembler::LoadLoad);
1839 
1840   __ profile_taken_branch(r0, r1);
1841   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1842                              InvocationCounter::counter_offset();
1843   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1844                               InvocationCounter::counter_offset();
1845 
1846   // load branch displacement
1847   if (!is_wide) {
1848     __ ldrh(r2, at_bcp(1));
1849     __ rev16(r2, r2);
1850     // sign extend the 16 bit value in r2
1851     __ sbfm(r2, r2, 0, 15);
1852   } else {
1853     __ ldrw(r2, at_bcp(1));
1854     __ revw(r2, r2);
1855     // sign extend the 32 bit value in r2
1856     __ sbfm(r2, r2, 0, 31);
1857   }
1858 
1859   // Handle all the JSR stuff here, then exit.
1860   // It's much shorter and cleaner than intermingling with the non-JSR
1861   // normal-branch stuff occurring below.
1862 
1863   if (is_jsr) {
1864     // Pre-load the next target bytecode into rscratch1
1865     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1866     // compute return address as bci
1867     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1868     __ add(rscratch2, rscratch2,
1869            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1870     __ sub(r1, rbcp, rscratch2);
1871     __ push_i(r1);
1872     // Adjust the bcp by the 16-bit displacement in r2
1873     __ add(rbcp, rbcp, r2);
1874     __ dispatch_only(vtos, /*generate_poll*/true);
1875     return;
1876   }
1877 
1878   // Normal (non-jsr) branch handling
1879 
1880   // Adjust the bcp by the displacement in r2
1881   __ add(rbcp, rbcp, r2);
1882 
1883   assert(UseLoopCounter || !UseOnStackReplacement,
1884          "on-stack-replacement requires loop counters");
1885   Label backedge_counter_overflow;
1886   Label profile_method;
1887   Label dispatch;
1888   if (UseLoopCounter) {
1889     // increment backedge counter for backward branches
1890     // r0: MDO
1891     // w1: MDO bumped taken-count
1892     // r2: target offset
1893     __ cmp(r2, zr);
1894     __ br(Assembler::GT, dispatch); // count only if backward branch
1895 
1896     // ECN: FIXME: This code smells
1897     // check if MethodCounters exists
1898     Label has_counters;
1899     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1900     __ cbnz(rscratch1, has_counters);
1901     __ push(r0);
1902     __ push(r1);
1903     __ push(r2);
1904     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1905             InterpreterRuntime::build_method_counters), rmethod);
1906     __ pop(r2);
1907     __ pop(r1);
1908     __ pop(r0);
1909     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1910     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1911     __ bind(has_counters);
1912 
1913     if (TieredCompilation) {
1914       Label no_mdo;
1915       int increment = InvocationCounter::count_increment;
1916       if (ProfileInterpreter) {
1917         // Are we profiling?
1918         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1919         __ cbz(r1, no_mdo);
1920         // Increment the MDO backedge counter
1921         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1922                                            in_bytes(InvocationCounter::counter_offset()));
1923         const Address mask(r1, in_bytes(MethodData::backedge_mask_offset()));
1924         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1925                                    r0, rscratch1, false, Assembler::EQ,
1926                                    UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1927         __ b(dispatch);
1928       }
1929       __ bind(no_mdo);
1930       // Increment backedge counter in MethodCounters*
1931       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1932       const Address mask(rscratch1, in_bytes(MethodCounters::backedge_mask_offset()));
1933       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1934                                  r0, rscratch2, false, Assembler::EQ,
1935                                  UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1936     } else { // not TieredCompilation
1937       // increment counter
1938       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1939       __ ldrw(r0, Address(rscratch2, be_offset));        // load backedge counter
1940       __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1941       __ strw(rscratch1, Address(rscratch2, be_offset));        // store counter
1942 
1943       __ ldrw(r0, Address(rscratch2, inv_offset));    // load invocation counter
1944       __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1945       __ addw(r0, r0, rscratch1);        // add both counters
1946 
1947       if (ProfileInterpreter) {
1948         // Test to see if we should create a method data oop
1949         __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_profile_limit_offset())));
1950         __ cmpw(r0, rscratch1);
1951         __ br(Assembler::LT, dispatch);
1952 
1953         // if no method data exists, go to profile method
1954         __ test_method_data_pointer(r0, profile_method);
1955 
1956         if (UseOnStackReplacement) {
1957           // check for overflow against w1 which is the MDO taken count
1958           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1959           __ cmpw(r1, rscratch1);
1960           __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1961 
1962           // When ProfileInterpreter is on, the backedge_count comes
1963           // from the MethodData*, which value does not get reset on
1964           // the call to frequency_counter_overflow().  To avoid
1965           // excessive calls to the overflow routine while the method is
1966           // being compiled, add a second test to make sure the overflow
1967           // function is called only once every overflow_frequency.
1968           const int overflow_frequency = 1024;
1969           __ andsw(r1, r1, overflow_frequency - 1);
1970           __ br(Assembler::EQ, backedge_counter_overflow);
1971 
1972         }
1973       } else {
1974         if (UseOnStackReplacement) {
1975           // check for overflow against w0, which is the sum of the
1976           // counters
1977           __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1978           __ cmpw(r0, rscratch1);
1979           __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1980         }
1981       }
1982     }
1983     __ bind(dispatch);
1984   }
1985 
1986   // Pre-load the next target bytecode into rscratch1
1987   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1988 
1989   // continue with the bytecode @ target
1990   // rscratch1: target bytecode
1991   // rbcp: target bcp
1992   __ dispatch_only(vtos, /*generate_poll*/true);
1993 
1994   if (UseLoopCounter) {
1995     if (ProfileInterpreter) {
1996       // Out-of-line code to allocate method data oop.
1997       __ bind(profile_method);
1998       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1999       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
2000       __ set_method_data_pointer_for_bcp();
2001       __ b(dispatch);
2002     }
2003 
2004     if (UseOnStackReplacement) {
2005       // invocation counter overflow
2006       __ bind(backedge_counter_overflow);
2007       __ neg(r2, r2);
2008       __ add(r2, r2, rbcp);     // branch bcp
2009       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
2010       __ call_VM(noreg,
2011                  CAST_FROM_FN_PTR(address,
2012                                   InterpreterRuntime::frequency_counter_overflow),
2013                  r2);
2014       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
2015 
2016       // r0: osr nmethod (osr ok) or NULL (osr not possible)
2017       // w1: target bytecode
2018       // r2: scratch
2019       __ cbz(r0, dispatch);     // test result -- no osr if null
2020       // nmethod may have been invalidated (VM may block upon call_VM return)
2021       __ ldrb(r2, Address(r0, nmethod::state_offset()));
2022       if (nmethod::in_use != 0)
2023         __ sub(r2, r2, nmethod::in_use);
2024       __ cbnz(r2, dispatch);
2025 
2026       // We have the address of an on stack replacement routine in r0
2027       // We need to prepare to execute the OSR method. First we must
2028       // migrate the locals and monitors off of the stack.
2029 
2030       __ mov(r19, r0);                             // save the nmethod
2031 
2032       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2033 
2034       // r0 is OSR buffer, move it to expected parameter location
2035       __ mov(j_rarg0, r0);
2036 
2037       // remove activation
2038       // get sender esp
2039       __ ldr(esp,
2040           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
2041       // remove frame anchor
2042       __ leave();
2043       // Ensure compiled code always sees stack at proper alignment
2044       __ andr(sp, esp, -16);
2045 
2046       // and begin the OSR nmethod
2047       __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
2048       __ br(rscratch1);
2049     }
2050   }
2051 }
2052 
2053 
2054 void TemplateTable::if_0cmp(Condition cc)
2055 {
2056   transition(itos, vtos);
2057   // assume branch is more often taken than not (loops use backward branches)
2058   Label not_taken;
2059   if (cc == equal)
2060     __ cbnzw(r0, not_taken);
2061   else if (cc == not_equal)
2062     __ cbzw(r0, not_taken);
2063   else {
2064     __ andsw(zr, r0, r0);
2065     __ br(j_not(cc), not_taken);
2066   }
2067 
2068   branch(false, false);
2069   __ bind(not_taken);
2070   __ profile_not_taken_branch(r0);
2071 }
2072 
2073 void TemplateTable::if_icmp(Condition cc)
2074 {
2075   transition(itos, vtos);
2076   // assume branch is more often taken than not (loops use backward branches)
2077   Label not_taken;
2078   __ pop_i(r1);
2079   __ cmpw(r1, r0, Assembler::LSL);
2080   __ br(j_not(cc), not_taken);
2081   branch(false, false);
2082   __ bind(not_taken);
2083   __ profile_not_taken_branch(r0);
2084 }
2085 
2086 void TemplateTable::if_nullcmp(Condition cc)
2087 {
2088   transition(atos, vtos);
2089   // assume branch is more often taken than not (loops use backward branches)
2090   Label not_taken;
2091   if (cc == equal)
2092     __ cbnz(r0, not_taken);
2093   else
2094     __ cbz(r0, not_taken);
2095   branch(false, false);
2096   __ bind(not_taken);
2097   __ profile_not_taken_branch(r0);
2098 }
2099 
2100 void TemplateTable::if_acmp(Condition cc)
2101 {
2102   transition(atos, vtos);
2103   // assume branch is more often taken than not (loops use backward branches)
2104   Label taken, not_taken;
2105   __ pop_ptr(r1);
2106   __ cmpoop(r1, r0);
2107 
2108   if (EnableValhalla) { 
2109     __ br(Assembler::NE, (cc == not_equal) ? taken : not_taken);
2110     __ cbz(r1, (cc == equal) ? taken : not_taken);
2111     __ ldr(r2, Address(r1, oopDesc::mark_offset_in_bytes()));  
2112     // DMS CHECK: Is code below correct or we should use tbz? 
2113     __ andr(r2, r2, markOopDesc::always_locked_pattern && 0xF);
2114     __ cmp(r2, (u1) markOopDesc::always_locked_pattern);
2115     cc = (cc == equal) ? not_equal : equal;
2116   }
2117 
2118 
2119   __ br(j_not(cc), not_taken);
2120   __ bind(taken);
2121   branch(false, false);
2122   __ bind(not_taken);
2123   __ profile_not_taken_branch(r0);
2124 }
2125 
2126 void TemplateTable::ret() {
2127   transition(vtos, vtos);
2128   // We might be moving to a safepoint.  The thread which calls
2129   // Interpreter::notice_safepoints() will effectively flush its cache
2130   // when it makes a system call, but we need to do something to
2131   // ensure that we see the changed dispatch table.
2132   __ membar(MacroAssembler::LoadLoad);
2133 
2134   locals_index(r1);
2135   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2136   __ profile_ret(r1, r2);
2137   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2138   __ lea(rbcp, Address(rbcp, r1));
2139   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2140   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2141 }
2142 
2143 void TemplateTable::wide_ret() {
2144   transition(vtos, vtos);
2145   locals_index_wide(r1);
2146   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2147   __ profile_ret(r1, r2);
2148   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2149   __ lea(rbcp, Address(rbcp, r1));
2150   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2151   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2152 }
2153 
2154 
2155 void TemplateTable::tableswitch() {
2156   Label default_case, continue_execution;
2157   transition(itos, vtos);
2158   // align rbcp
2159   __ lea(r1, at_bcp(BytesPerInt));
2160   __ andr(r1, r1, -BytesPerInt);
2161   // load lo & hi
2162   __ ldrw(r2, Address(r1, BytesPerInt));
2163   __ ldrw(r3, Address(r1, 2 * BytesPerInt));
2164   __ rev32(r2, r2);
2165   __ rev32(r3, r3);
2166   // check against lo & hi
2167   __ cmpw(r0, r2);
2168   __ br(Assembler::LT, default_case);
2169   __ cmpw(r0, r3);
2170   __ br(Assembler::GT, default_case);
2171   // lookup dispatch offset
2172   __ subw(r0, r0, r2);
2173   __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2174   __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2175   __ profile_switch_case(r0, r1, r2);
2176   // continue execution
2177   __ bind(continue_execution);
2178   __ rev32(r3, r3);
2179   __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2180   __ add(rbcp, rbcp, r3, ext::sxtw);
2181   __ dispatch_only(vtos, /*generate_poll*/true);
2182   // handle default
2183   __ bind(default_case);
2184   __ profile_switch_default(r0);
2185   __ ldrw(r3, Address(r1, 0));
2186   __ b(continue_execution);
2187 }
2188 
2189 void TemplateTable::lookupswitch() {
2190   transition(itos, itos);
2191   __ stop("lookupswitch bytecode should have been rewritten");
2192 }
2193 
2194 void TemplateTable::fast_linearswitch() {
2195   transition(itos, vtos);
2196   Label loop_entry, loop, found, continue_execution;
2197   // bswap r0 so we can avoid bswapping the table entries
2198   __ rev32(r0, r0);
2199   // align rbcp
2200   __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2201                                     // this instruction (change offsets
2202                                     // below)
2203   __ andr(r19, r19, -BytesPerInt);
2204   // set counter
2205   __ ldrw(r1, Address(r19, BytesPerInt));
2206   __ rev32(r1, r1);
2207   __ b(loop_entry);
2208   // table search
2209   __ bind(loop);
2210   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2211   __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2212   __ cmpw(r0, rscratch1);
2213   __ br(Assembler::EQ, found);
2214   __ bind(loop_entry);
2215   __ subs(r1, r1, 1);
2216   __ br(Assembler::PL, loop);
2217   // default case
2218   __ profile_switch_default(r0);
2219   __ ldrw(r3, Address(r19, 0));
2220   __ b(continue_execution);
2221   // entry found -> get offset
2222   __ bind(found);
2223   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2224   __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2225   __ profile_switch_case(r1, r0, r19);
2226   // continue execution
2227   __ bind(continue_execution);
2228   __ rev32(r3, r3);
2229   __ add(rbcp, rbcp, r3, ext::sxtw);
2230   __ ldrb(rscratch1, Address(rbcp, 0));
2231   __ dispatch_only(vtos, /*generate_poll*/true);
2232 }
2233 
2234 void TemplateTable::fast_binaryswitch() {
2235   transition(itos, vtos);
2236   // Implementation using the following core algorithm:
2237   //
2238   // int binary_search(int key, LookupswitchPair* array, int n) {
2239   //   // Binary search according to "Methodik des Programmierens" by
2240   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2241   //   int i = 0;
2242   //   int j = n;
2243   //   while (i+1 < j) {
2244   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2245   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2246   //     // where a stands for the array and assuming that the (inexisting)
2247   //     // element a[n] is infinitely big.
2248   //     int h = (i + j) >> 1;
2249   //     // i < h < j
2250   //     if (key < array[h].fast_match()) {
2251   //       j = h;
2252   //     } else {
2253   //       i = h;
2254   //     }
2255   //   }
2256   //   // R: a[i] <= key < a[i+1] or Q
2257   //   // (i.e., if key is within array, i is the correct index)
2258   //   return i;
2259   // }
2260 
2261   // Register allocation
2262   const Register key   = r0; // already set (tosca)
2263   const Register array = r1;
2264   const Register i     = r2;
2265   const Register j     = r3;
2266   const Register h     = rscratch1;
2267   const Register temp  = rscratch2;
2268 
2269   // Find array start
2270   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2271                                           // get rid of this
2272                                           // instruction (change
2273                                           // offsets below)
2274   __ andr(array, array, -BytesPerInt);
2275 
2276   // Initialize i & j
2277   __ mov(i, 0);                            // i = 0;
2278   __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2279 
2280   // Convert j into native byteordering
2281   __ rev32(j, j);
2282 
2283   // And start
2284   Label entry;
2285   __ b(entry);
2286 
2287   // binary search loop
2288   {
2289     Label loop;
2290     __ bind(loop);
2291     // int h = (i + j) >> 1;
2292     __ addw(h, i, j);                           // h = i + j;
2293     __ lsrw(h, h, 1);                                   // h = (i + j) >> 1;
2294     // if (key < array[h].fast_match()) {
2295     //   j = h;
2296     // } else {
2297     //   i = h;
2298     // }
2299     // Convert array[h].match to native byte-ordering before compare
2300     __ ldr(temp, Address(array, h, Address::lsl(3)));
2301     __ rev32(temp, temp);
2302     __ cmpw(key, temp);
2303     // j = h if (key <  array[h].fast_match())
2304     __ csel(j, h, j, Assembler::LT);
2305     // i = h if (key >= array[h].fast_match())
2306     __ csel(i, h, i, Assembler::GE);
2307     // while (i+1 < j)
2308     __ bind(entry);
2309     __ addw(h, i, 1);          // i+1
2310     __ cmpw(h, j);             // i+1 < j
2311     __ br(Assembler::LT, loop);
2312   }
2313 
2314   // end of binary search, result index is i (must check again!)
2315   Label default_case;
2316   // Convert array[i].match to native byte-ordering before compare
2317   __ ldr(temp, Address(array, i, Address::lsl(3)));
2318   __ rev32(temp, temp);
2319   __ cmpw(key, temp);
2320   __ br(Assembler::NE, default_case);
2321 
2322   // entry found -> j = offset
2323   __ add(j, array, i, ext::uxtx, 3);
2324   __ ldrw(j, Address(j, BytesPerInt));
2325   __ profile_switch_case(i, key, array);
2326   __ rev32(j, j);
2327   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2328   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2329   __ dispatch_only(vtos, /*generate_poll*/true);
2330 
2331   // default case -> j = default offset
2332   __ bind(default_case);
2333   __ profile_switch_default(i);
2334   __ ldrw(j, Address(array, -2 * BytesPerInt));
2335   __ rev32(j, j);
2336   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2337   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2338   __ dispatch_only(vtos, /*generate_poll*/true);
2339 }
2340 
2341 
2342 void TemplateTable::_return(TosState state)
2343 {
2344   transition(state, state);
2345   assert(_desc->calls_vm(),
2346          "inconsistent calls_vm information"); // call in remove_activation
2347 
2348   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2349     assert(state == vtos, "only valid state");
2350 
2351     __ ldr(c_rarg1, aaddress(0));
2352     __ load_klass(r3, c_rarg1);
2353     __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2354     Label skip_register_finalizer;
2355     __ tbz(r3, exact_log2(JVM_ACC_HAS_FINALIZER), skip_register_finalizer);
2356 
2357     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2358 
2359     __ bind(skip_register_finalizer);
2360   }
2361 
2362   // Issue a StoreStore barrier after all stores but before return
2363   // from any constructor for any class with a final field.  We don't
2364   // know if this is a finalizer, so we always do so.
2365   if (_desc->bytecode() == Bytecodes::_return)
2366     __ membar(MacroAssembler::StoreStore);
2367 
2368   // Narrow result if state is itos but result type is smaller.
2369   // Need to narrow in the return bytecode rather than in generate_return_entry
2370   // since compiled code callers expect the result to already be narrowed.
2371   if (state == itos) {
2372     __ narrow(r0);
2373   }
2374 
2375   __ remove_activation(state); 
2376   __ ret(lr);
2377 }
2378 
2379 // ----------------------------------------------------------------------------
2380 // Volatile variables demand their effects be made known to all CPU's
2381 // in order.  Store buffers on most chips allow reads & writes to
2382 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2383 // without some kind of memory barrier (i.e., it's not sufficient that
2384 // the interpreter does not reorder volatile references, the hardware
2385 // also must not reorder them).
2386 //
2387 // According to the new Java Memory Model (JMM):
2388 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2389 //     writes act as aquire & release, so:
2390 // (2) A read cannot let unrelated NON-volatile memory refs that
2391 //     happen after the read float up to before the read.  It's OK for
2392 //     non-volatile memory refs that happen before the volatile read to
2393 //     float down below it.
2394 // (3) Similar a volatile write cannot let unrelated NON-volatile
2395 //     memory refs that happen BEFORE the write float down to after the
2396 //     write.  It's OK for non-volatile memory refs that happen after the
2397 //     volatile write to float up before it.
2398 //
2399 // We only put in barriers around volatile refs (they are expensive),
2400 // not _between_ memory refs (that would require us to track the
2401 // flavor of the previous memory refs).  Requirements (2) and (3)
2402 // require some barriers before volatile stores and after volatile
2403 // loads.  These nearly cover requirement (1) but miss the
2404 // volatile-store-volatile-load case.  This final case is placed after
2405 // volatile-stores although it could just as well go before
2406 // volatile-loads.
2407 
2408 void TemplateTable::resolve_cache_and_index(int byte_no,
2409                                             Register Rcache,
2410                                             Register index,
2411                                             size_t index_size) {
2412   const Register temp = r19;
2413   assert_different_registers(Rcache, index, temp);
2414 
2415   Label resolved;
2416 
2417   Bytecodes::Code code = bytecode();
2418   switch (code) {
2419   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2420   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2421   default: break;
2422   }
2423 
2424   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2425   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2426   __ subs(zr, temp, (int) code);  // have we resolved this bytecode?
2427   __ br(Assembler::EQ, resolved);
2428 
2429   // resolve first time through
2430   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2431   __ mov(temp, (int) code);
2432   __ call_VM(noreg, entry, temp);
2433 
2434   // Update registers with resolved info
2435   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2436   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2437   // so all clients ofthis method must be modified accordingly
2438   __ bind(resolved);
2439 }
2440 
2441 // The Rcache and index registers must be set before call
2442 // n.b unlike x86 cache already includes the index offset
2443 void TemplateTable::load_field_cp_cache_entry(Register obj,
2444                                               Register cache,
2445                                               Register index,
2446                                               Register off,
2447                                               Register flags,
2448                                               bool is_static = false) {
2449   assert_different_registers(cache, index, flags, off);
2450 
2451   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2452   // Field offset
2453   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2454                                           ConstantPoolCacheEntry::f2_offset())));
2455   // Flags
2456   __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2457                                            ConstantPoolCacheEntry::flags_offset())));
2458 
2459   // klass overwrite register
2460   if (is_static) {
2461     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2462                                         ConstantPoolCacheEntry::f1_offset())));
2463     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2464     __ ldr(obj, Address(obj, mirror_offset));
2465     __ resolve_oop_handle(obj);
2466   }
2467 }
2468 
2469 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2470                                                Register method,
2471                                                Register itable_index,
2472                                                Register flags,
2473                                                bool is_invokevirtual,
2474                                                bool is_invokevfinal, /*unused*/
2475                                                bool is_invokedynamic) {
2476   // setup registers
2477   const Register cache = rscratch2;
2478   const Register index = r4;
2479   assert_different_registers(method, flags);
2480   assert_different_registers(method, cache, index);
2481   assert_different_registers(itable_index, flags);
2482   assert_different_registers(itable_index, cache, index);
2483   // determine constant pool cache field offsets
2484   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2485   const int method_offset = in_bytes(
2486     ConstantPoolCache::base_offset() +
2487       (is_invokevirtual
2488        ? ConstantPoolCacheEntry::f2_offset()
2489        : ConstantPoolCacheEntry::f1_offset()));
2490   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2491                                     ConstantPoolCacheEntry::flags_offset());
2492   // access constant pool cache fields
2493   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2494                                     ConstantPoolCacheEntry::f2_offset());
2495 
2496   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2497   resolve_cache_and_index(byte_no, cache, index, index_size);
2498   __ ldr(method, Address(cache, method_offset));
2499 
2500   if (itable_index != noreg) {
2501     __ ldr(itable_index, Address(cache, index_offset));
2502   }
2503   __ ldrw(flags, Address(cache, flags_offset));
2504 }
2505 
2506 
2507 // The registers cache and index expected to be set before call.
2508 // Correct values of the cache and index registers are preserved.
2509 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2510                                             bool is_static, bool has_tos) {
2511   // do the JVMTI work here to avoid disturbing the register state below
2512   // We use c_rarg registers here because we want to use the register used in
2513   // the call to the VM
2514   if (JvmtiExport::can_post_field_access()) {
2515     // Check to see if a field access watch has been set before we
2516     // take the time to call into the VM.
2517     Label L1;
2518     assert_different_registers(cache, index, r0);
2519     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2520     __ ldrw(r0, Address(rscratch1));
2521     __ cbzw(r0, L1);
2522 
2523     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2524     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2525 
2526     if (is_static) {
2527       __ mov(c_rarg1, zr); // NULL object reference
2528     } else {
2529       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2530       __ verify_oop(c_rarg1);
2531     }
2532     // c_rarg1: object pointer or NULL
2533     // c_rarg2: cache entry pointer
2534     // c_rarg3: jvalue object on the stack
2535     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2536                                        InterpreterRuntime::post_field_access),
2537                c_rarg1, c_rarg2, c_rarg3);
2538     __ get_cache_and_index_at_bcp(cache, index, 1);
2539     __ bind(L1);
2540   }
2541 }
2542 
2543 void TemplateTable::pop_and_check_object(Register r)
2544 {
2545   __ pop_ptr(r);
2546   __ null_check(r);  // for field access must check obj.
2547   __ verify_oop(r);
2548 }
2549 
2550 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2551 {
2552   const Register cache = r2;
2553   const Register index = r3;
2554   const Register obj   = r4;
2555   const Register off   = r19;
2556   const Register flags = r0;
2557   const Register raw_flags = r6;
2558   const Register bc    = r4; // uses same reg as obj, so don't mix them
2559 
2560   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2561   jvmti_post_field_access(cache, index, is_static, false);
2562   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2563 
2564   if (!is_static) {
2565     // obj is on the stack
2566     pop_and_check_object(obj);
2567   }
2568 
2569   // 8179954: We need to make sure that the code generated for
2570   // volatile accesses forms a sequentially-consistent set of
2571   // operations when combined with STLR and LDAR.  Without a leading
2572   // membar it's possible for a simple Dekker test to fail if loads
2573   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2574   // the stores in one method and we interpret the loads in another.
2575   if (! UseBarriersForVolatile) {
2576     Label notVolatile;
2577     __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2578     __ membar(MacroAssembler::AnyAny);
2579     __ bind(notVolatile);
2580   }
2581 
2582   const Address field(obj, off);
2583 
2584   Label Done, notByte, notBool, notInt, notShort, notChar,
2585               notLong, notFloat, notObj, notDouble;
2586 
2587   // x86 uses a shift and mask or wings it with a shift plus assert
2588   // the mask is not needed. aarch64 just uses bitfield extract
2589   __ ubfxw(flags, raw_flags, ConstantPoolCacheEntry::tos_state_shift, ConstantPoolCacheEntry::tos_state_bits);

2590 
2591   assert(btos == 0, "change code, btos != 0");
2592   __ cbnz(flags, notByte);
2593 
2594   // Don't rewrite getstatic, only getfield
2595   if (is_static) rc = may_not_rewrite;
2596 
2597   // btos
2598   __ access_load_at(T_BYTE, IN_HEAP, r0, field, noreg, noreg);
2599   __ push(btos);
2600   // Rewrite bytecode to be faster
2601   if (rc == may_rewrite) {
2602     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2603   }
2604   __ b(Done);
2605 
2606   __ bind(notByte);
2607   __ cmp(flags, (u1)ztos);
2608   __ br(Assembler::NE, notBool);
2609 
2610   // ztos (same code as btos)
2611   __ access_load_at(T_BOOLEAN, IN_HEAP, r0, field, noreg, noreg);
2612   __ push(ztos);
2613   // Rewrite bytecode to be faster
2614   if (rc == may_rewrite) {
2615     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2616     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2617   }
2618   __ b(Done);
2619 
2620   __ bind(notBool);
2621   __ cmp(flags, (u1)atos);
2622   __ br(Assembler::NE, notObj);
2623   // atos
2624   if (!EnableValhalla) {
2625     do_oop_load(_masm, field, r0, IN_HEAP);
2626     __ push(atos);
2627     if (rc == may_rewrite) {
2628       patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2629     }  
2630     __ b(Done);
2631   } else { // Valhalla
2632 
2633     if (is_static) {
2634       __ load_heap_oop(r0, field);
2635       Label isFlattenable, isUninitialized;
2636       // Issue below if the static field has not been initialized yet
2637       __ test_field_is_flattenable(raw_flags, r10, isFlattenable);
2638         // Not flattenable case
2639         __ push(atos);
2640         __ b(Done);
2641       // Flattenable case, must not return null even if uninitialized
2642       __ bind(isFlattenable);
2643         __ cbz(r0, isUninitialized);
2644           __ push(atos);
2645           __ b(Done);
2646         __ bind(isUninitialized);
2647           __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2648           __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_value_field), obj, raw_flags);
2649           __ verify_oop(r0);
2650           __ push(atos);
2651           __ b(Done);
2652     } else {
2653       Label isFlattened, isInitialized, isFlattenable, rewriteFlattenable;
2654         __ test_field_is_flattenable(raw_flags, r10, isFlattenable);
2655         // Non-flattenable field case, also covers the object case
2656         __ load_heap_oop(r0, field);
2657         __ push(atos);
2658         if (rc == may_rewrite) {
2659           patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2660         }
2661         __ b(Done);
2662       __ bind(isFlattenable);
2663         __ test_field_is_flattened(raw_flags, r10, isFlattened);
2664          // Non-flattened field case
2665           __ load_heap_oop(r0, field);
2666           __ cbnz(r0, isInitialized);
2667             __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2668             __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_instance_value_field), obj, raw_flags);
2669           __ bind(isInitialized);
2670           __ verify_oop(r0);
2671           __ push(atos);
2672           __ b(rewriteFlattenable);
2673         __ bind(isFlattened);
2674           __ ldr(r10, Address(cache, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset())));
2675           __ andw(raw_flags, raw_flags, ConstantPoolCacheEntry::field_index_mask);
2676           call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), obj, raw_flags, r10); 
2677           __ verify_oop(r0);
2678           __ push(atos);
2679       __ bind(rewriteFlattenable);
2680       if (rc == may_rewrite) { 
2681          patch_bytecode(Bytecodes::_fast_qgetfield, bc, r1);
2682       }
2683       __ b(Done);
2684     }
2685   }
2686 
2687   __ bind(notObj);
2688   __ cmp(flags, (u1)itos);
2689   __ br(Assembler::NE, notInt);
2690   // itos
2691   __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
2692   __ push(itos);
2693   // Rewrite bytecode to be faster
2694   if (rc == may_rewrite) {
2695     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2696   }
2697   __ b(Done);
2698 
2699   __ bind(notInt);
2700   __ cmp(flags, (u1)ctos);
2701   __ br(Assembler::NE, notChar);
2702   // ctos
2703   __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
2704   __ push(ctos);
2705   // Rewrite bytecode to be faster
2706   if (rc == may_rewrite) {
2707     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2708   }
2709   __ b(Done);
2710 
2711   __ bind(notChar);
2712   __ cmp(flags, (u1)stos);
2713   __ br(Assembler::NE, notShort);
2714   // stos
2715   __ access_load_at(T_SHORT, IN_HEAP, r0, field, noreg, noreg);
2716   __ push(stos);
2717   // Rewrite bytecode to be faster
2718   if (rc == may_rewrite) {
2719     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2720   }
2721   __ b(Done);
2722 
2723   __ bind(notShort);
2724   __ cmp(flags, (u1)ltos);
2725   __ br(Assembler::NE, notLong);
2726   // ltos
2727   __ access_load_at(T_LONG, IN_HEAP, r0, field, noreg, noreg);
2728   __ push(ltos);
2729   // Rewrite bytecode to be faster
2730   if (rc == may_rewrite) {
2731     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2732   }
2733   __ b(Done);
2734 
2735   __ bind(notLong);
2736   __ cmp(flags, (u1)ftos);
2737   __ br(Assembler::NE, notFloat);
2738   // ftos
2739   __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2740   __ push(ftos);
2741   // Rewrite bytecode to be faster
2742   if (rc == may_rewrite) {
2743     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2744   }
2745   __ b(Done);
2746 
2747   __ bind(notFloat);
2748 #ifdef ASSERT
2749   __ cmp(flags, (u1)dtos);
2750   __ br(Assembler::NE, notDouble);
2751 #endif
2752   // dtos
2753   __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2754   __ push(dtos);
2755   // Rewrite bytecode to be faster
2756   if (rc == may_rewrite) {
2757     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2758   }
2759 #ifdef ASSERT
2760   __ b(Done);
2761 
2762   __ bind(notDouble);
2763   __ stop("Bad state");
2764 #endif
2765 
2766   __ bind(Done);
2767 
2768   Label notVolatile;
2769   __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2770   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2771   __ bind(notVolatile);
2772 }
2773 
2774 
2775 void TemplateTable::getfield(int byte_no)
2776 {
2777   getfield_or_static(byte_no, false);
2778 }
2779 
2780 void TemplateTable::nofast_getfield(int byte_no) {
2781   getfield_or_static(byte_no, false, may_not_rewrite);
2782 }
2783 
2784 void TemplateTable::getstatic(int byte_no)
2785 {
2786   getfield_or_static(byte_no, true);
2787 }
2788 
2789 // The registers cache and index expected to be set before call.
2790 // The function may destroy various registers, just not the cache and index registers.
2791 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2792   transition(vtos, vtos);
2793 
2794   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2795 
2796   if (JvmtiExport::can_post_field_modification()) {
2797     // Check to see if a field modification watch has been set before
2798     // we take the time to call into the VM.
2799     Label L1;
2800     assert_different_registers(cache, index, r0);
2801     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2802     __ ldrw(r0, Address(rscratch1));
2803     __ cbz(r0, L1);
2804 
2805     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2806 
2807     if (is_static) {
2808       // Life is simple.  Null out the object pointer.
2809       __ mov(c_rarg1, zr);
2810     } else {
2811       // Life is harder. The stack holds the value on top, followed by
2812       // the object.  We don't know the size of the value, though; it
2813       // could be one or two words depending on its type. As a result,
2814       // we must find the type to determine where the object is.
2815       __ ldrw(c_rarg3, Address(c_rarg2,
2816                                in_bytes(cp_base_offset +
2817                                         ConstantPoolCacheEntry::flags_offset())));
2818       __ lsr(c_rarg3, c_rarg3,
2819              ConstantPoolCacheEntry::tos_state_shift);
2820       ConstantPoolCacheEntry::verify_tos_state_shift();
2821       Label nope2, done, ok;
2822       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2823       __ cmpw(c_rarg3, ltos);
2824       __ br(Assembler::EQ, ok);
2825       __ cmpw(c_rarg3, dtos);
2826       __ br(Assembler::NE, nope2);
2827       __ bind(ok);
2828       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2829       __ bind(nope2);
2830     }
2831     // cache entry pointer
2832     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2833     // object (tos)
2834     __ mov(c_rarg3, esp);
2835     // c_rarg1: object pointer set up above (NULL if static)
2836     // c_rarg2: cache entry pointer
2837     // c_rarg3: jvalue object on the stack
2838     __ call_VM(noreg,
2839                CAST_FROM_FN_PTR(address,
2840                                 InterpreterRuntime::post_field_modification),
2841                c_rarg1, c_rarg2, c_rarg3);
2842     __ get_cache_and_index_at_bcp(cache, index, 1);
2843     __ bind(L1);
2844   }
2845 }
2846 
2847 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2848   transition(vtos, vtos);
2849 
2850   const Register cache = r2;
2851   const Register index = r3;
2852   const Register obj   = r2;
2853   const Register off   = r19;
2854   const Register flags = r0;
2855   const Register flags2 = r6;
2856   const Register bc    = r4;
2857 
2858   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2859   jvmti_post_field_mod(cache, index, is_static);
2860   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2861 
2862   Label Done;
2863   __ mov(r5, flags);
2864 
2865   {
2866     Label notVolatile;
2867     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2868     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2869     __ bind(notVolatile);
2870   }
2871 
2872   // field address
2873   const Address field(obj, off);
2874 
2875   Label notByte, notBool, notInt, notShort, notChar,
2876         notLong, notFloat, notObj, notDouble;
2877 
2878   __ mov(flags2, flags); 
2879 
2880   // x86 uses a shift and mask or wings it with a shift plus assert
2881   // the mask is not needed. aarch64 just uses bitfield extract
2882   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2883 
2884   assert(btos == 0, "change code, btos != 0");
2885   __ cbnz(flags, notByte);
2886 
2887   // Don't rewrite putstatic, only putfield
2888   if (is_static) rc = may_not_rewrite;
2889 
2890   // btos
2891   {
2892     __ pop(btos);
2893     if (!is_static) pop_and_check_object(obj);
2894     __ access_store_at(T_BYTE, IN_HEAP, field, r0, noreg, noreg);
2895     if (rc == may_rewrite) {
2896       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2897     }
2898     __ b(Done);
2899   }
2900 
2901   __ bind(notByte);
2902   __ cmp(flags, (u1)ztos);
2903   __ br(Assembler::NE, notBool);
2904 
2905   // ztos
2906   {
2907     __ pop(ztos);
2908     if (!is_static) pop_and_check_object(obj);
2909     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg);
2910     if (rc == may_rewrite) {
2911       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2912     }
2913     __ b(Done);
2914   }
2915 
2916   __ bind(notBool);
2917   __ cmp(flags, (u1)atos);
2918   __ br(Assembler::NE, notObj);
2919 
2920   // atos
2921   {
2922      if (!EnableValhalla) {
2923       __ pop(atos);
2924       if (!is_static) pop_and_check_object(obj);
2925       // Store into the field
2926       do_oop_store(_masm, field, r0, IN_HEAP);
2927       if (rc == may_rewrite) {
2928         patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2929       }
2930       __ b(Done);
2931      } else { // Valhalla
2932 
2933       __ pop(atos);
2934       if (is_static) {
2935         Label notFlattenable;
2936          __ test_field_is_not_flattenable(flags2, r10, notFlattenable);
2937          __ null_check(r0);
2938          __ bind(notFlattenable);
2939          do_oop_store(_masm, field, r0, IN_HEAP);
2940          __ b(Done);
2941       } else {
2942         Label isFlattenable, isFlattened, notBuffered, notBuffered2, rewriteNotFlattenable, rewriteFlattenable;
2943         __ test_field_is_flattenable(flags2, r10, isFlattenable);
2944         // Not flattenable case, covers not flattenable values and objects
2945         pop_and_check_object(obj);
2946         // Store into the field
2947         do_oop_store(_masm, field, r0, IN_HEAP);
2948         __ bind(rewriteNotFlattenable);
2949         if (rc == may_rewrite) {
2950           patch_bytecode(Bytecodes::_fast_aputfield, bc, r19, true, byte_no); 
2951         }
2952         __ b(Done);
2953         // Implementation of the flattenable semantic
2954         __ bind(isFlattenable);
2955         __ null_check(r0);
2956         __ test_field_is_flattened(flags2, r10, isFlattened);
2957         // Not flattened case
2958         pop_and_check_object(obj);
2959         // Store into the field
2960         do_oop_store(_masm, field, r0, IN_HEAP);
2961         __ b(rewriteFlattenable);
2962         __ bind(isFlattened);
2963         pop_and_check_object(obj);
2964         call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flattened_value), r0, off, obj);
2965         __ bind(rewriteFlattenable);
2966         if (rc == may_rewrite) {
2967           patch_bytecode(Bytecodes::_fast_qputfield, bc, r19, true, byte_no);
2968         }
2969         __ b(Done);
2970       }
2971      }  // Valhalla
2972   }
2973 
2974   __ bind(notObj);
2975   __ cmp(flags, (u1)itos);
2976   __ br(Assembler::NE, notInt);
2977 
2978   // itos
2979   {
2980     __ pop(itos);
2981     if (!is_static) pop_and_check_object(obj);
2982     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg);
2983     if (rc == may_rewrite) {
2984       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
2985     }
2986     __ b(Done);
2987   }
2988 
2989   __ bind(notInt);
2990   __ cmp(flags, (u1)ctos);
2991   __ br(Assembler::NE, notChar);
2992 
2993   // ctos
2994   {
2995     __ pop(ctos);
2996     if (!is_static) pop_and_check_object(obj);
2997     __ access_store_at(T_CHAR, IN_HEAP, field, r0, noreg, noreg);
2998     if (rc == may_rewrite) {
2999       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
3000     }
3001     __ b(Done);
3002   }
3003 
3004   __ bind(notChar);
3005   __ cmp(flags, (u1)stos);
3006   __ br(Assembler::NE, notShort);
3007 
3008   // stos
3009   {
3010     __ pop(stos);
3011     if (!is_static) pop_and_check_object(obj);
3012     __ access_store_at(T_SHORT, IN_HEAP, field, r0, noreg, noreg);
3013     if (rc == may_rewrite) {
3014       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
3015     }
3016     __ b(Done);
3017   }
3018 
3019   __ bind(notShort);
3020   __ cmp(flags, (u1)ltos);
3021   __ br(Assembler::NE, notLong);
3022 
3023   // ltos
3024   {
3025     __ pop(ltos);
3026     if (!is_static) pop_and_check_object(obj);
3027     __ access_store_at(T_LONG, IN_HEAP, field, r0, noreg, noreg);
3028     if (rc == may_rewrite) {
3029       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
3030     }
3031     __ b(Done);
3032   }
3033 
3034   __ bind(notLong);
3035   __ cmp(flags, (u1)ftos);
3036   __ br(Assembler::NE, notFloat);
3037 
3038   // ftos
3039   {
3040     __ pop(ftos);
3041     if (!is_static) pop_and_check_object(obj);
3042     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
3043     if (rc == may_rewrite) {
3044       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
3045     }
3046     __ b(Done);
3047   }
3048 
3049   __ bind(notFloat);
3050 #ifdef ASSERT
3051   __ cmp(flags, (u1)dtos);
3052   __ br(Assembler::NE, notDouble);
3053 #endif
3054 
3055   // dtos
3056   {
3057     __ pop(dtos);
3058     if (!is_static) pop_and_check_object(obj);
3059     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
3060     if (rc == may_rewrite) {
3061       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
3062     }
3063   }
3064 
3065 #ifdef ASSERT
3066   __ b(Done);
3067 
3068   __ bind(notDouble);
3069   __ stop("Bad state");
3070 #endif
3071 
3072   __ bind(Done);
3073 
3074   {
3075     Label notVolatile;
3076     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3077     __ membar(MacroAssembler::StoreLoad);
3078     __ bind(notVolatile);
3079   }
3080 }
3081 
3082 void TemplateTable::putfield(int byte_no)
3083 {
3084   putfield_or_static(byte_no, false);
3085 }
3086 
3087 void TemplateTable::nofast_putfield(int byte_no) {
3088   putfield_or_static(byte_no, false, may_not_rewrite);
3089 }
3090 
3091 void TemplateTable::putstatic(int byte_no) {
3092   putfield_or_static(byte_no, true);
3093 }
3094 
3095 void TemplateTable::jvmti_post_fast_field_mod()
3096 {
3097   if (JvmtiExport::can_post_field_modification()) {
3098     // Check to see if a field modification watch has been set before
3099     // we take the time to call into the VM.
3100     Label L2;
3101     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3102     __ ldrw(c_rarg3, Address(rscratch1));
3103     __ cbzw(c_rarg3, L2);
3104     __ pop_ptr(r19);                  // copy the object pointer from tos
3105     __ verify_oop(r19);
3106     __ push_ptr(r19);                 // put the object pointer back on tos
3107     // Save tos values before call_VM() clobbers them. Since we have
3108     // to do it for every data type, we use the saved values as the
3109     // jvalue object.
3110     switch (bytecode()) {          // load values into the jvalue object
3111     case Bytecodes::_fast_qputfield: //fall through
3112     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
3113     case Bytecodes::_fast_bputfield: // fall through
3114     case Bytecodes::_fast_zputfield: // fall through
3115     case Bytecodes::_fast_sputfield: // fall through
3116     case Bytecodes::_fast_cputfield: // fall through
3117     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
3118     case Bytecodes::_fast_dputfield: __ push_d(); break;
3119     case Bytecodes::_fast_fputfield: __ push_f(); break;
3120     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
3121 
3122     default:
3123       ShouldNotReachHere();
3124     }
3125     __ mov(c_rarg3, esp);             // points to jvalue on the stack
3126     // access constant pool cache entry
3127     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
3128     __ verify_oop(r19);
3129     // r19: object pointer copied above
3130     // c_rarg2: cache entry pointer
3131     // c_rarg3: jvalue object on the stack
3132     __ call_VM(noreg,
3133                CAST_FROM_FN_PTR(address,
3134                                 InterpreterRuntime::post_field_modification),
3135                r19, c_rarg2, c_rarg3);
3136 
3137     switch (bytecode()) {             // restore tos values
3138     case Bytecodes::_fast_qputfield: //fall through
3139     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
3140     case Bytecodes::_fast_bputfield: // fall through
3141     case Bytecodes::_fast_zputfield: // fall through
3142     case Bytecodes::_fast_sputfield: // fall through
3143     case Bytecodes::_fast_cputfield: // fall through
3144     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
3145     case Bytecodes::_fast_dputfield: __ pop_d(); break;
3146     case Bytecodes::_fast_fputfield: __ pop_f(); break;
3147     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
3148     default: break;
3149     }
3150     __ bind(L2);
3151   }
3152 }
3153 
3154 void TemplateTable::fast_storefield(TosState state)
3155 {
3156   transition(state, vtos);
3157 
3158   ByteSize base = ConstantPoolCache::base_offset();
3159 
3160   jvmti_post_fast_field_mod();
3161 
3162   // access constant pool cache
3163   __ get_cache_and_index_at_bcp(r2, r1, 1);
3164 
3165   // test for volatile with r3
3166   __ ldrw(r3, Address(r2, in_bytes(base +
3167                                    ConstantPoolCacheEntry::flags_offset())));
3168 
3169   // replace index with field offset from cache entry
3170   __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
3171 
3172   {
3173     Label notVolatile;
3174     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3175     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
3176     __ bind(notVolatile);
3177   }
3178 
3179   Label notVolatile;
3180 
3181   // Get object from stack
3182   pop_and_check_object(r2);
3183 
3184   // field address
3185   const Address field(r2, r1);
3186 
3187   // access field
3188   switch (bytecode()) {
3189   case Bytecodes::_fast_qputfield: //fall through 
3190    {
3191       Label isFlattened, done; 
3192       __ null_check(r0);
3193       __ test_field_is_flattened(r3, r10, isFlattened);
3194       // No Flattened case
3195       do_oop_store(_masm, field, r0, IN_HEAP);
3196       __ b(done);
3197       __ bind(isFlattened);
3198       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flattened_value), r0, r1, r2);
3199       __ bind(done);
3200     }
3201     break;
3202   case Bytecodes::_fast_aputfield:
3203     do_oop_store(_masm, field, r0, IN_HEAP);
3204     break;
3205   case Bytecodes::_fast_lputfield:
3206     __ access_store_at(T_LONG, IN_HEAP, field, r0, noreg, noreg);
3207     break;
3208   case Bytecodes::_fast_iputfield:
3209     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg);
3210     break;
3211   case Bytecodes::_fast_zputfield:
3212     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg);
3213     break;
3214   case Bytecodes::_fast_bputfield:
3215     __ access_store_at(T_BYTE, IN_HEAP, field, r0, noreg, noreg);
3216     break;
3217   case Bytecodes::_fast_sputfield:
3218     __ access_store_at(T_SHORT, IN_HEAP, field, r0, noreg, noreg);
3219     break;
3220   case Bytecodes::_fast_cputfield:
3221     __ access_store_at(T_CHAR, IN_HEAP, field, r0, noreg, noreg);
3222     break;
3223   case Bytecodes::_fast_fputfield:
3224     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
3225     break;
3226   case Bytecodes::_fast_dputfield:
3227     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
3228     break;
3229   default:
3230     ShouldNotReachHere();
3231   }
3232 
3233   {
3234     Label notVolatile;
3235     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3236     __ membar(MacroAssembler::StoreLoad);
3237     __ bind(notVolatile);
3238   }
3239 }
3240 
3241 
3242 void TemplateTable::fast_accessfield(TosState state)
3243 {
3244   transition(atos, state);
3245   // Do the JVMTI work here to avoid disturbing the register state below
3246   if (JvmtiExport::can_post_field_access()) {
3247     // Check to see if a field access watch has been set before we
3248     // take the time to call into the VM.
3249     Label L1;
3250     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3251     __ ldrw(r2, Address(rscratch1));
3252     __ cbzw(r2, L1);
3253     // access constant pool cache entry
3254     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
3255     __ verify_oop(r0);
3256     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
3257     __ mov(c_rarg1, r0);
3258     // c_rarg1: object pointer copied above
3259     // c_rarg2: cache entry pointer
3260     __ call_VM(noreg,
3261                CAST_FROM_FN_PTR(address,
3262                                 InterpreterRuntime::post_field_access),
3263                c_rarg1, c_rarg2);
3264     __ pop_ptr(r0); // restore object pointer
3265     __ bind(L1);
3266   }
3267 
3268   // access constant pool cache
3269   __ get_cache_and_index_at_bcp(r2, r1, 1);
3270   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3271                                   ConstantPoolCacheEntry::f2_offset())));
3272   __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3273                                    ConstantPoolCacheEntry::flags_offset())));
3274 
3275   // r0: object
3276   __ verify_oop(r0);
3277   __ null_check(r0);
3278   const Address field(r0, r1);
3279 
3280   // 8179954: We need to make sure that the code generated for
3281   // volatile accesses forms a sequentially-consistent set of
3282   // operations when combined with STLR and LDAR.  Without a leading
3283   // membar it's possible for a simple Dekker test to fail if loads
3284   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3285   // the stores in one method and we interpret the loads in another.
3286   if (! UseBarriersForVolatile) {
3287     Label notVolatile;
3288     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3289     __ membar(MacroAssembler::AnyAny);
3290     __ bind(notVolatile);
3291   }
3292 
3293   // access field
3294   switch (bytecode()) {
3295   case Bytecodes::_fast_qgetfield: 
3296     {
3297        Label isFlattened, isInitialized, Done;
3298        // DMS CHECK: We don't need to reload multiple times, but stay close to original code
3299        __ ldrw(r10, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()))); 
3300        __ test_field_is_flattened(r10, r10, isFlattened);
3301         // Non-flattened field case
3302         __ mov(r10, r0);
3303         __ load_heap_oop(r0, field);
3304         __ cbnz(r0, isInitialized);
3305           __ mov(r0, r10);
3306           __ ldrw(r10, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()))); 
3307           __ andw(r10, r10, ConstantPoolCacheEntry::field_index_mask);
3308           __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_instance_value_field), r0, r10);
3309         __ bind(isInitialized);
3310         __ verify_oop(r0);
3311         __ b(Done);
3312       __ bind(isFlattened);
3313         __ ldrw(r10, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset())));
3314         __ andw(r10, r10, ConstantPoolCacheEntry::field_index_mask);
3315         __ ldr(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset())));
3316         call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), r0, r10, r3);
3317         __ verify_oop(r0);
3318       __ bind(Done);
3319     }
3320     break;
3321   case Bytecodes::_fast_agetfield:
3322     do_oop_load(_masm, field, r0, IN_HEAP);
3323     __ verify_oop(r0);
3324     break;
3325   case Bytecodes::_fast_lgetfield:
3326     __ access_load_at(T_LONG, IN_HEAP, r0, field, noreg, noreg);
3327     break;
3328   case Bytecodes::_fast_igetfield:
3329     __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
3330     break;
3331   case Bytecodes::_fast_bgetfield:
3332     __ access_load_at(T_BYTE, IN_HEAP, r0, field, noreg, noreg);
3333     break;
3334   case Bytecodes::_fast_sgetfield:
3335     __ access_load_at(T_SHORT, IN_HEAP, r0, field, noreg, noreg);
3336     break;
3337   case Bytecodes::_fast_cgetfield:
3338     __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
3339     break;
3340   case Bytecodes::_fast_fgetfield:
3341     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3342     break;
3343   case Bytecodes::_fast_dgetfield:
3344     __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg);
3345     break;
3346   default:
3347     ShouldNotReachHere();
3348   }
3349   {
3350     Label notVolatile;
3351     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3352     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3353     __ bind(notVolatile);
3354   }
3355 }
3356 
3357 void TemplateTable::fast_xaccess(TosState state)
3358 {
3359   transition(vtos, state);
3360 
3361   // get receiver
3362   __ ldr(r0, aaddress(0));
3363   // access constant pool cache
3364   __ get_cache_and_index_at_bcp(r2, r3, 2);
3365   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3366                                   ConstantPoolCacheEntry::f2_offset())));
3367 
3368   // 8179954: We need to make sure that the code generated for
3369   // volatile accesses forms a sequentially-consistent set of
3370   // operations when combined with STLR and LDAR.  Without a leading
3371   // membar it's possible for a simple Dekker test to fail if loads
3372   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3373   // the stores in one method and we interpret the loads in another.
3374   if (! UseBarriersForVolatile) {
3375     Label notVolatile;
3376     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3377                                      ConstantPoolCacheEntry::flags_offset())));
3378     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3379     __ membar(MacroAssembler::AnyAny);
3380     __ bind(notVolatile);
3381   }
3382 
3383   // make sure exception is reported in correct bcp range (getfield is
3384   // next instruction)
3385   __ increment(rbcp);
3386   __ null_check(r0);
3387   switch (state) {
3388   case itos:
3389     __ access_load_at(T_INT, IN_HEAP, r0, Address(r0, r1, Address::lsl(0)), noreg, noreg);
3390     break;
3391   case atos:
3392     do_oop_load(_masm, Address(r0, r1, Address::lsl(0)), r0, IN_HEAP);
3393     __ verify_oop(r0);
3394     break;
3395   case ftos:
3396     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, Address(r0, r1, Address::lsl(0)), noreg, noreg);
3397     break;
3398   default:
3399     ShouldNotReachHere();
3400   }
3401 
3402   {
3403     Label notVolatile;
3404     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3405                                      ConstantPoolCacheEntry::flags_offset())));
3406     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3407     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3408     __ bind(notVolatile);
3409   }
3410 
3411   __ decrement(rbcp);
3412 }
3413 
3414 
3415 
3416 //-----------------------------------------------------------------------------
3417 // Calls
3418 
3419 void TemplateTable::count_calls(Register method, Register temp)
3420 {
3421   __ call_Unimplemented();
3422 }
3423 
3424 void TemplateTable::prepare_invoke(int byte_no,
3425                                    Register method, // linked method (or i-klass)
3426                                    Register index,  // itable index, MethodType, etc.
3427                                    Register recv,   // if caller wants to see it
3428                                    Register flags   // if caller wants to test it
3429                                    ) {
3430   // determine flags
3431   Bytecodes::Code code = bytecode();
3432   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3433   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3434   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3435   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3436   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3437   const bool load_receiver       = (recv  != noreg);
3438   const bool save_flags          = (flags != noreg);
3439   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3440   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3441   assert(flags == noreg || flags == r3, "");
3442   assert(recv  == noreg || recv  == r2, "");
3443 
3444   // setup registers & access constant pool cache
3445   if (recv  == noreg)  recv  = r2;
3446   if (flags == noreg)  flags = r3;
3447   assert_different_registers(method, index, recv, flags);
3448 
3449   // save 'interpreter return address'
3450   __ save_bcp();
3451 
3452   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3453 
3454   // maybe push appendix to arguments (just before return address)
3455   if (is_invokedynamic || is_invokehandle) {
3456     Label L_no_push;
3457     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3458     // Push the appendix as a trailing parameter.
3459     // This must be done before we get the receiver,
3460     // since the parameter_size includes it.
3461     __ push(r19);
3462     __ mov(r19, index);
3463     __ load_resolved_reference_at_index(index, r19);
3464     __ pop(r19);
3465     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3466     __ bind(L_no_push);
3467   }
3468 
3469   // load receiver if needed (note: no return address pushed yet)
3470   if (load_receiver) {
3471     __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3472     // FIXME -- is this actually correct? looks like it should be 2
3473     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3474     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3475     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3476     // __ movptr(recv, recv_addr);
3477     __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3478     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3479     __ verify_oop(recv);
3480   }
3481 
3482   // compute return type
3483   // x86 uses a shift and mask or wings it with a shift plus assert
3484   // the mask is not needed. aarch64 just uses bitfield extract
3485   __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3486   // load return address
3487   {
3488     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3489     __ mov(rscratch1, table_addr);
3490     __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3491   }
3492 }
3493 
3494 
3495 void TemplateTable::invokevirtual_helper(Register index,
3496                                          Register recv,
3497                                          Register flags)
3498 {
3499   // Uses temporary registers r0, r3
3500   assert_different_registers(index, recv, r0, r3);
3501   // Test for an invoke of a final method
3502   Label notFinal;
3503   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3504 
3505   const Register method = index;  // method must be rmethod
3506   assert(method == rmethod,
3507          "methodOop must be rmethod for interpreter calling convention");
3508 
3509   // do the call - the index is actually the method to call
3510   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3511 
3512   // It's final, need a null check here!
3513   __ null_check(recv);
3514 
3515   // profile this call
3516   __ profile_final_call(r0);
3517   __ profile_arguments_type(r0, method, r4, true);
3518 
3519   __ jump_from_interpreted(method, r0);
3520 
3521   __ bind(notFinal);
3522 
3523   // get receiver klass
3524   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3525   __ load_klass(r0, recv);
3526 
3527   // profile this call
3528   __ profile_virtual_call(r0, rlocals, r3);
3529 
3530   // get target methodOop & entry point
3531   __ lookup_virtual_method(r0, index, method);
3532   __ profile_arguments_type(r3, method, r4, true);
3533   // FIXME -- this looks completely redundant. is it?
3534   // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3535   __ jump_from_interpreted(method, r3);
3536 }
3537 
3538 void TemplateTable::invokevirtual(int byte_no)
3539 {
3540   transition(vtos, vtos);
3541   assert(byte_no == f2_byte, "use this argument");
3542 
3543   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3544 
3545   // rmethod: index (actually a Method*)
3546   // r2: receiver
3547   // r3: flags
3548 
3549   invokevirtual_helper(rmethod, r2, r3);
3550 }
3551 
3552 void TemplateTable::invokespecial(int byte_no)
3553 {
3554   transition(vtos, vtos);
3555   assert(byte_no == f1_byte, "use this argument");
3556 
3557   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3558                  r2);  // get receiver also for null check
3559   __ verify_oop(r2);
3560   __ null_check(r2);
3561   // do the call
3562   __ profile_call(r0);
3563   __ profile_arguments_type(r0, rmethod, rbcp, false);
3564   __ jump_from_interpreted(rmethod, r0);
3565 }
3566 
3567 void TemplateTable::invokestatic(int byte_no)
3568 {
3569   transition(vtos, vtos);
3570   assert(byte_no == f1_byte, "use this argument");
3571 
3572   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3573   // do the call
3574   __ profile_call(r0);
3575   __ profile_arguments_type(r0, rmethod, r4, false);
3576   __ jump_from_interpreted(rmethod, r0);
3577 }
3578 
3579 void TemplateTable::fast_invokevfinal(int byte_no)
3580 {
3581   __ call_Unimplemented();
3582 }
3583 
3584 void TemplateTable::invokeinterface(int byte_no) {
3585   transition(vtos, vtos);
3586   assert(byte_no == f1_byte, "use this argument");
3587 
3588   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 Method*
3589                  r2, r3); // recv, flags
3590 
3591   // r0: interface klass (from f1)
3592   // rmethod: method (from f2)
3593   // r2: receiver
3594   // r3: flags
3595 
3596   // First check for Object case, then private interface method,
3597   // then regular interface method.
3598 
3599   // Special case of invokeinterface called for virtual method of
3600   // java.lang.Object.  See cpCache.cpp for details.
3601   Label notObjectMethod;
3602   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notObjectMethod);
3603 
3604   invokevirtual_helper(rmethod, r2, r3);
3605   __ bind(notObjectMethod);
3606 
3607   Label no_such_interface;
3608 
3609   // Check for private method invocation - indicated by vfinal
3610   Label notVFinal;
3611   __ tbz(r3, ConstantPoolCacheEntry::is_vfinal_shift, notVFinal);
3612 
3613   // Get receiver klass into r3 - also a null check
3614   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3615   __ load_klass(r3, r2);
3616 
3617   Label subtype;
3618   __ check_klass_subtype(r3, r0, r4, subtype);
3619   // If we get here the typecheck failed
3620   __ b(no_such_interface);
3621   __ bind(subtype);
3622 
3623   __ profile_final_call(r0);
3624   __ profile_arguments_type(r0, rmethod, r4, true);
3625   __ jump_from_interpreted(rmethod, r0);
3626 
3627   __ bind(notVFinal);
3628 
3629   // Get receiver klass into r3 - also a null check
3630   __ restore_locals();
3631   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3632   __ load_klass(r3, r2);
3633 
3634   Label no_such_method;
3635 
3636   // Preserve method for throw_AbstractMethodErrorVerbose.
3637   __ mov(r16, rmethod);
3638   // Receiver subtype check against REFC.
3639   // Superklass in r0. Subklass in r3. Blows rscratch2, r13
3640   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3641                              r3, r0, noreg,
3642                              // outputs: scan temp. reg, scan temp. reg
3643                              rscratch2, r13,
3644                              no_such_interface,
3645                              /*return_method=*/false);
3646 
3647   // profile this call
3648   __ profile_virtual_call(r3, r13, r19);
3649 
3650   // Get declaring interface class from method, and itable index
3651   __ ldr(r0, Address(rmethod, Method::const_offset()));
3652   __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
3653   __ ldr(r0, Address(r0, ConstantPool::pool_holder_offset_in_bytes()));
3654   __ ldrw(rmethod, Address(rmethod, Method::itable_index_offset()));
3655   __ subw(rmethod, rmethod, Method::itable_index_max);
3656   __ negw(rmethod, rmethod);
3657 
3658   // Preserve recvKlass for throw_AbstractMethodErrorVerbose.
3659   __ mov(rlocals, r3);
3660   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3661                              rlocals, r0, rmethod,
3662                              // outputs: method, scan temp. reg
3663                              rmethod, r13,
3664                              no_such_interface);
3665 
3666   // rmethod,: methodOop to call
3667   // r2: receiver
3668   // Check for abstract method error
3669   // Note: This should be done more efficiently via a throw_abstract_method_error
3670   //       interpreter entry point and a conditional jump to it in case of a null
3671   //       method.
3672   __ cbz(rmethod, no_such_method);
3673 
3674   __ profile_arguments_type(r3, rmethod, r13, true);
3675 
3676   // do the call
3677   // r2: receiver
3678   // rmethod,: methodOop
3679   __ jump_from_interpreted(rmethod, r3);
3680   __ should_not_reach_here();
3681 
3682   // exception handling code follows...
3683   // note: must restore interpreter registers to canonical
3684   //       state for exception handling to work correctly!
3685 
3686   __ bind(no_such_method);
3687   // throw exception
3688   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3689   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3690   // Pass arguments for generating a verbose error message.
3691   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose), r3, r16);
3692   // the call_VM checks for exception, so we should never return here.
3693   __ should_not_reach_here();
3694 
3695   __ bind(no_such_interface);
3696   // throw exception
3697   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3698   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3699   // Pass arguments for generating a verbose error message.
3700   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3701                    InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose), r3, r0);
3702   // the call_VM checks for exception, so we should never return here.
3703   __ should_not_reach_here();
3704   return;
3705 }
3706 
3707 void TemplateTable::invokehandle(int byte_no) {
3708   transition(vtos, vtos);
3709   assert(byte_no == f1_byte, "use this argument");
3710 
3711   prepare_invoke(byte_no, rmethod, r0, r2);
3712   __ verify_method_ptr(r2);
3713   __ verify_oop(r2);
3714   __ null_check(r2);
3715 
3716   // FIXME: profile the LambdaForm also
3717 
3718   // r13 is safe to use here as a scratch reg because it is about to
3719   // be clobbered by jump_from_interpreted().
3720   __ profile_final_call(r13);
3721   __ profile_arguments_type(r13, rmethod, r4, true);
3722 
3723   __ jump_from_interpreted(rmethod, r0);
3724 }
3725 
3726 void TemplateTable::invokedynamic(int byte_no) {
3727   transition(vtos, vtos);
3728   assert(byte_no == f1_byte, "use this argument");
3729 
3730   prepare_invoke(byte_no, rmethod, r0);
3731 
3732   // r0: CallSite object (from cpool->resolved_references[])
3733   // rmethod: MH.linkToCallSite method (from f2)
3734 
3735   // Note:  r0_callsite is already pushed by prepare_invoke
3736 
3737   // %%% should make a type profile for any invokedynamic that takes a ref argument
3738   // profile this call
3739   __ profile_call(rbcp);
3740   __ profile_arguments_type(r3, rmethod, r13, false);
3741 
3742   __ verify_oop(r0);
3743 
3744   __ jump_from_interpreted(rmethod, r0);
3745 }
3746 
3747 
3748 //-----------------------------------------------------------------------------
3749 // Allocation
3750 
3751 void TemplateTable::_new() {
3752   transition(vtos, atos);
3753 
3754   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3755   Label slow_case;
3756   Label done;
3757   Label initialize_header;
3758   Label initialize_object; // including clearing the fields
3759 
3760   __ get_cpool_and_tags(r4, r0);
3761   // Make sure the class we're about to instantiate has been resolved.
3762   // This is done before loading InstanceKlass to be consistent with the order
3763   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3764   const int tags_offset = Array<u1>::base_offset_in_bytes();
3765   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3766   __ lea(rscratch1, Address(rscratch1, tags_offset));
3767   __ ldarb(rscratch1, rscratch1);
3768   __ cmp(rscratch1, (u1)JVM_CONSTANT_Class);
3769   __ br(Assembler::NE, slow_case);
3770 
3771   // get InstanceKlass
3772   __ load_resolved_klass_at_offset(r4, r3, r4, rscratch1);
3773 
3774   // make sure klass is initialized & doesn't have finalizer
3775   // make sure klass is fully initialized
3776   __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3777   __ cmp(rscratch1, (u1)InstanceKlass::fully_initialized);
3778   __ br(Assembler::NE, slow_case);
3779 
3780   // get instance_size in InstanceKlass (scaled to a count of bytes)
3781   __ ldrw(r3,
3782           Address(r4,
3783                   Klass::layout_helper_offset()));
3784   // test to see if it has a finalizer or is malformed in some way
3785   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3786 
3787   // Allocate the instance:
3788   //  If TLAB is enabled:
3789   //    Try to allocate in the TLAB.
3790   //    If fails, go to the slow path.
3791   //  Else If inline contiguous allocations are enabled:
3792   //    Try to allocate in eden.
3793   //    If fails due to heap end, go to slow path.
3794   //
3795   //  If TLAB is enabled OR inline contiguous is enabled:
3796   //    Initialize the allocation.
3797   //    Exit.
3798   //
3799   //  Go to slow path.
3800   const bool allow_shared_alloc =
3801     Universe::heap()->supports_inline_contig_alloc();
3802 
3803   if (UseTLAB) {
3804     __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case);
3805 
3806     if (ZeroTLAB) {
3807       // the fields have been already cleared
3808       __ b(initialize_header);
3809     } else {
3810       // initialize both the header and fields
3811       __ b(initialize_object);
3812     }
3813   } else {
3814     // Allocation in the shared Eden, if allowed.
3815     //
3816     // r3: instance size in bytes
3817     if (allow_shared_alloc) {
3818       __ eden_allocate(r0, r3, 0, r10, slow_case);
3819     }
3820   }
3821 
3822   // If UseTLAB or allow_shared_alloc are true, the object is created above and
3823   // there is an initialize need. Otherwise, skip and go to the slow path.
3824   if (UseTLAB || allow_shared_alloc) {
3825     // The object is initialized before the header.  If the object size is
3826     // zero, go directly to the header initialization.
3827     __ bind(initialize_object);
3828     __ sub(r3, r3, sizeof(oopDesc));
3829     __ cbz(r3, initialize_header);
3830 
3831     // Initialize object fields
3832     {
3833       __ add(r2, r0, sizeof(oopDesc));
3834       Label loop;
3835       __ bind(loop);
3836       __ str(zr, Address(__ post(r2, BytesPerLong)));
3837       __ sub(r3, r3, BytesPerLong);
3838       __ cbnz(r3, loop);
3839     }
3840 
3841     // initialize object header only.
3842     __ bind(initialize_header);
3843     if (UseBiasedLocking) {
3844       __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3845     } else {
3846       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3847     }
3848     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3849     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3850     __ store_klass(r0, r4);      // store klass last
3851 
3852     {
3853       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3854       // Trigger dtrace event for fastpath
3855       __ push(atos); // save the return value
3856       __ call_VM_leaf(
3857            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3858       __ pop(atos); // restore the return value
3859 
3860     }
3861     __ b(done);
3862   }
3863 
3864   // slow case
3865   __ bind(slow_case);
3866   __ get_constant_pool(c_rarg1);
3867   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3868   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3869   __ verify_oop(r0);
3870 
3871   // continue
3872   __ bind(done);
3873   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3874   __ membar(Assembler::StoreStore);
3875 }
3876 
3877 void TemplateTable::defaultvalue() {
3878   transition(vtos, atos);
3879   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3880   __ get_constant_pool(c_rarg1);
3881   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::defaultvalue),
3882           c_rarg1, c_rarg2);
3883   __ verify_oop(r0);
3884   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3885   __ membar(Assembler::StoreStore);
3886 }
3887 
3888 void TemplateTable::withfield() {
3889   transition(vtos, atos);
3890   resolve_cache_and_index(f2_byte, c_rarg1 /*cache*/, c_rarg2 /*index*/, sizeof(u2));
3891 
3892   // n.b. unlike x86 cache is now rcpool plus the indexed offset
3893   // so using rcpool to meet shared code expectations
3894  
3895   call_VM(r1, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), rcpool);
3896   __ verify_oop(r1);
3897   __ add(esp, esp, r0);
3898   __ mov(r0, r1);
3899 }
3900 
3901 void TemplateTable::newarray() {
3902   transition(itos, atos);
3903   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3904   __ mov(c_rarg2, r0);
3905   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3906           c_rarg1, c_rarg2);
3907   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3908   __ membar(Assembler::StoreStore);
3909 }
3910 
3911 void TemplateTable::anewarray() {
3912   transition(itos, atos);
3913   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3914   __ get_constant_pool(c_rarg1);
3915   __ mov(c_rarg3, r0);
3916   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3917           c_rarg1, c_rarg2, c_rarg3);
3918   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3919   __ membar(Assembler::StoreStore);
3920 }
3921 
3922 void TemplateTable::arraylength() {
3923   transition(atos, itos);
3924   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3925   __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3926 }
3927 
3928 void TemplateTable::checkcast()
3929 {
3930   transition(atos, atos);
3931   Label done, is_null, ok_is_subtype, quicked, resolved;
3932   __ cbz(r0, is_null);
3933 
3934   // Get cpool & tags index
3935   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3936   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3937   // See if bytecode has already been quicked
3938   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3939   __ lea(r1, Address(rscratch1, r19));
3940   __ ldarb(r1, r1);
3941   __ cmp(r1, (u1)JVM_CONSTANT_Class);
3942   __ br(Assembler::EQ, quicked);
3943 
3944   __ push(atos); // save receiver for result, and for GC
3945   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3946   // vm_result_2 has metadata result
3947   __ get_vm_result_2(r0, rthread);
3948   __ pop(r3); // restore receiver
3949   __ b(resolved);
3950 
3951   // Get superklass in r0 and subklass in r3
3952   __ bind(quicked);
3953   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3954   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1); // r0 = klass
3955 
3956   __ bind(resolved);
3957   __ load_klass(r19, r3);
3958 
3959   // Generate subtype check.  Blows r2, r5.  Object in r3.
3960   // Superklass in r0.  Subklass in r19.
3961   __ gen_subtype_check(r19, ok_is_subtype);
3962 
3963   // Come here on failure
3964   __ push(r3);
3965   // object is at TOS
3966   __ b(Interpreter::_throw_ClassCastException_entry);
3967 
3968   // Come here on success
3969   __ bind(ok_is_subtype);
3970   __ mov(r0, r3); // Restore object in r3
3971 


3972   __ b(done);
3973   __ bind(is_null);
3974 
3975   // Collect counts on whether this test sees NULLs a lot or not.
3976   if (ProfileInterpreter) {
3977     __ profile_null_seen(r2);


3978   }
3979 
3980   if (EnableValhalla) {
3981       // Get cpool & tags index
3982       __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3983       __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3984       // See if bytecode has already been quicked
3985       __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3986       __ lea(r1, Address(rscratch1, r19));
3987       __ ldarb(r1, r1);
3988     // See if CP entry is a Q-descriptor
3989     __ andr (r1, r1, JVM_CONSTANT_QDESC_BIT);
3990     __ cmp(r1, (u1) JVM_CONSTANT_QDESC_BIT);
3991     __ br(Assembler::NE, done);
3992     __ b(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
3993   }
3994 
3995   __ bind(done);
3996 }
3997 
3998 void TemplateTable::instanceof() {
3999   transition(atos, itos);
4000   Label done, is_null, ok_is_subtype, quicked, resolved;
4001   __ cbz(r0, is_null);
4002 
4003   // Get cpool & tags index
4004   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
4005   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
4006   // See if bytecode has already been quicked
4007   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
4008   __ lea(r1, Address(rscratch1, r19));
4009   __ ldarb(r1, r1);
4010   __ cmp(r1, (u1)JVM_CONSTANT_Class);
4011   __ br(Assembler::EQ, quicked);
4012 
4013   __ push(atos); // save receiver for result, and for GC
4014   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4015   // vm_result_2 has metadata result
4016   __ get_vm_result_2(r0, rthread);
4017   __ pop(r3); // restore receiver
4018   __ verify_oop(r3);
4019   __ load_klass(r3, r3);
4020   __ b(resolved);
4021 
4022   // Get superklass in r0 and subklass in r3
4023   __ bind(quicked);
4024   __ load_klass(r3, r0);
4025   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1);
4026 
4027   __ bind(resolved);
4028 
4029   // Generate subtype check.  Blows r2, r5
4030   // Superklass in r0.  Subklass in r3.
4031   __ gen_subtype_check(r3, ok_is_subtype);
4032 
4033   // Come here on failure
4034   __ mov(r0, 0);
4035   __ b(done);
4036   // Come here on success
4037   __ bind(ok_is_subtype);
4038   __ mov(r0, 1);
4039 
4040   // Collect counts on whether this test sees NULLs a lot or not.
4041   if (ProfileInterpreter) {
4042     __ b(done);
4043     __ bind(is_null);
4044     __ profile_null_seen(r2);
4045   } else {
4046     __ bind(is_null);   // same as 'done'
4047   }
4048   __ bind(done);
4049   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
4050   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
4051 }
4052 
4053 //-----------------------------------------------------------------------------
4054 // Breakpoints
4055 void TemplateTable::_breakpoint() {
4056   // Note: We get here even if we are single stepping..
4057   // jbug inists on setting breakpoints at every bytecode
4058   // even if we are in single step mode.
4059 
4060   transition(vtos, vtos);
4061 
4062   // get the unpatched byte code
4063   __ get_method(c_rarg1);
4064   __ call_VM(noreg,
4065              CAST_FROM_FN_PTR(address,
4066                               InterpreterRuntime::get_original_bytecode_at),
4067              c_rarg1, rbcp);
4068   __ mov(r19, r0);
4069 
4070   // post the breakpoint event
4071   __ call_VM(noreg,
4072              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
4073              rmethod, rbcp);
4074 
4075   // complete the execution of original bytecode
4076   __ mov(rscratch1, r19);
4077   __ dispatch_only_normal(vtos);
4078 }
4079 
4080 //-----------------------------------------------------------------------------
4081 // Exceptions
4082 
4083 void TemplateTable::athrow() {
4084   transition(atos, vtos);
4085   __ null_check(r0);
4086   __ b(Interpreter::throw_exception_entry());
4087 }
4088 
4089 //-----------------------------------------------------------------------------
4090 // Synchronization
4091 //
4092 // Note: monitorenter & exit are symmetric routines; which is reflected
4093 //       in the assembly code structure as well
4094 //
4095 // Stack layout:
4096 //
4097 // [expressions  ] <--- esp               = expression stack top
4098 // ..
4099 // [expressions  ]
4100 // [monitor entry] <--- monitor block top = expression stack bot
4101 // ..
4102 // [monitor entry]
4103 // [frame data   ] <--- monitor block bot
4104 // ...
4105 // [saved rbp    ] <--- rbp
4106 void TemplateTable::monitorenter()
4107 {
4108   transition(atos, vtos);
4109 
4110   // check for NULL object
4111   __ null_check(r0);
4112 
4113   __ resolve(IS_NOT_NULL, r0);
4114 
4115   const Address monitor_block_top(
4116         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4117   const Address monitor_block_bot(
4118         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4119   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4120 
4121   Label allocated;
4122 
4123   // initialize entry pointer
4124   __ mov(c_rarg1, zr); // points to free slot or NULL
4125 
4126   // find a free slot in the monitor block (result in c_rarg1)
4127   {
4128     Label entry, loop, exit;
4129     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
4130                                         // starting with top-most entry
4131     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4132 
4133     __ b(entry);
4134 
4135     __ bind(loop);
4136     // check if current entry is used
4137     // if not used then remember entry in c_rarg1
4138     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
4139     __ cmp(zr, rscratch1);
4140     __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
4141     // check if current entry is for same object
4142     __ cmp(r0, rscratch1);
4143     // if same object then stop searching
4144     __ br(Assembler::EQ, exit);
4145     // otherwise advance to next entry
4146     __ add(c_rarg3, c_rarg3, entry_size);
4147     __ bind(entry);
4148     // check if bottom reached
4149     __ cmp(c_rarg3, c_rarg2);
4150     // if not at bottom then check this entry
4151     __ br(Assembler::NE, loop);
4152     __ bind(exit);
4153   }
4154 
4155   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
4156                             // if found, continue with that on
4157 
4158   // allocate one if there's no free slot
4159   {
4160     Label entry, loop;
4161     // 1. compute new pointers            // rsp: old expression stack top
4162     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
4163     __ sub(esp, esp, entry_size);         // move expression stack top
4164     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
4165     __ mov(c_rarg3, esp);                 // set start value for copy loop
4166     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
4167 
4168     __ sub(sp, sp, entry_size);           // make room for the monitor
4169 
4170     __ b(entry);
4171     // 2. move expression stack contents
4172     __ bind(loop);
4173     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
4174                                                    // word from old location
4175     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
4176     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
4177     __ bind(entry);
4178     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
4179     __ br(Assembler::NE, loop);      // if not at bottom then
4180                                      // copy next word
4181   }
4182 
4183   // call run-time routine
4184   // c_rarg1: points to monitor entry
4185   __ bind(allocated);
4186 
4187   // Increment bcp to point to the next bytecode, so exception
4188   // handling for async. exceptions work correctly.
4189   // The object has already been poped from the stack, so the
4190   // expression stack looks correct.
4191   __ increment(rbcp);
4192 
4193   // store object
4194   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
4195   __ lock_object(c_rarg1);
4196 
4197   // check to make sure this monitor doesn't cause stack overflow after locking
4198   __ save_bcp();  // in case of exception
4199   __ generate_stack_overflow_check(0);
4200 
4201   // The bcp has already been incremented. Just need to dispatch to
4202   // next instruction.
4203   __ dispatch_next(vtos);
4204 }
4205 
4206 
4207 void TemplateTable::monitorexit()
4208 {
4209   transition(atos, vtos);
4210 
4211   // check for NULL object
4212   __ null_check(r0);
4213 
4214   __ resolve(IS_NOT_NULL, r0);
4215 
4216   const Address monitor_block_top(
4217         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4218   const Address monitor_block_bot(
4219         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4220   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4221 
4222   Label found;
4223 
4224   // find matching slot
4225   {
4226     Label entry, loop;
4227     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
4228                                         // starting with top-most entry
4229     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4230                                         // of monitor block
4231     __ b(entry);
4232 
4233     __ bind(loop);
4234     // check if current entry is for same object
4235     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
4236     __ cmp(r0, rscratch1);
4237     // if same object then stop searching
4238     __ br(Assembler::EQ, found);
4239     // otherwise advance to next entry
4240     __ add(c_rarg1, c_rarg1, entry_size);
4241     __ bind(entry);
4242     // check if bottom reached
4243     __ cmp(c_rarg1, c_rarg2);
4244     // if not at bottom then check this entry
4245     __ br(Assembler::NE, loop);
4246   }
4247 
4248   // error handling. Unlocking was not block-structured
4249   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4250                    InterpreterRuntime::throw_illegal_monitor_state_exception));
4251   __ should_not_reach_here();
4252 
4253   // call run-time routine
4254   __ bind(found);
4255   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
4256   __ unlock_object(c_rarg1);
4257   __ pop_ptr(r0); // discard object
4258 }
4259 
4260 
4261 // Wide instructions
4262 void TemplateTable::wide()
4263 {
4264   __ load_unsigned_byte(r19, at_bcp(1));
4265   __ mov(rscratch1, (address)Interpreter::_wentry_point);
4266   __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
4267   __ br(rscratch1);
4268 }
4269 
4270 
4271 // Multi arrays
4272 void TemplateTable::multianewarray() {
4273   transition(vtos, atos);
4274   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
4275   // last dim is on top of stack; we want address of first one:
4276   // first_addr = last_addr + (ndims - 1) * wordSize
4277   __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
4278   __ sub(c_rarg1, c_rarg1, wordSize);
4279   call_VM(r0,
4280           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
4281           c_rarg1);
4282   __ load_unsigned_byte(r1, at_bcp(3));
4283   __ lea(esp, Address(esp, r1, Address::uxtw(3)));
4284 }
--- EOF ---