1 /*
   2  * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "gc/shared/barrierSet.inline.hpp"
  28 #include "gc/shared/cardTable.hpp"
  29 #include "gc/shared/cardTableBarrierSet.inline.hpp"
  30 #include "gc/shared/collectedHeap.hpp"
  31 #include "interp_masm_arm.hpp"
  32 #include "interpreter/interpreter.hpp"
  33 #include "interpreter/interpreterRuntime.hpp"
  34 #include "logging/log.hpp"
  35 #include "oops/arrayOop.hpp"
  36 #include "oops/markOop.hpp"
  37 #include "oops/method.hpp"
  38 #include "oops/methodData.hpp"
  39 #include "prims/jvmtiExport.hpp"
  40 #include "prims/jvmtiThreadState.hpp"
  41 #include "runtime/basicLock.hpp"
  42 #include "runtime/biasedLocking.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 
  45 #if INCLUDE_ALL_GCS
  46 #include "gc/g1/g1BarrierSet.hpp"
  47 #include "gc/g1/g1CollectedHeap.inline.hpp"
  48 #include "gc/g1/heapRegion.hpp"
  49 #endif // INCLUDE_ALL_GCS
  50 
  51 //--------------------------------------------------------------------
  52 // Implementation of InterpreterMacroAssembler
  53 
  54 
  55 
  56 
  57 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) {
  58 }
  59 
  60 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
  61 #if defined(ASSERT) && !defined(AARCH64)
  62   // Ensure that last_sp is not filled.
  63   { Label L;
  64     ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
  65     cbz(Rtemp, L);
  66     stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL");
  67     bind(L);
  68   }
  69 #endif // ASSERT && !AARCH64
  70 
  71   // Rbcp must be saved/restored since it may change due to GC.
  72   save_bcp();
  73 
  74 #ifdef AARCH64
  75   check_no_cached_stack_top(Rtemp);
  76   save_stack_top();
  77   check_extended_sp(Rtemp);
  78   cut_sp_before_call();
  79 #endif // AARCH64
  80 
  81   // super call
  82   MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions);
  83 
  84 #ifdef AARCH64
  85   // Restore SP to extended SP
  86   restore_sp_after_call(Rtemp);
  87   check_stack_top();
  88   clear_cached_stack_top();
  89 #endif // AARCH64
  90 
  91   // Restore interpreter specific registers.
  92   restore_bcp();
  93   restore_method();
  94 }
  95 
  96 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  97   assert(entry, "Entry must have been generated by now");
  98   b(entry);
  99 }
 100 
 101 void InterpreterMacroAssembler::check_and_handle_popframe() {
 102   if (can_pop_frame()) {
 103     Label L;
 104     const Register popframe_cond = R2_tmp;
 105 
 106     // Initiate popframe handling only if it is not already being processed.  If the flag
 107     // has the popframe_processing bit set, it means that this code is called *during* popframe
 108     // handling - we don't want to reenter.
 109 
 110     ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset()));
 111     tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L);
 112     tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L);
 113 
 114     // Call Interpreter::remove_activation_preserving_args_entry() to get the
 115     // address of the same-named entrypoint in the generated interpreter code.
 116     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
 117 
 118     // Call indirectly to avoid generation ordering problem.
 119     jump(R0);
 120 
 121     bind(L);
 122   }
 123 }
 124 
 125 
 126 // Blows R2, Rtemp. Sets TOS cached value.
 127 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
 128   const Register thread_state = R2_tmp;
 129 
 130   ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
 131 
 132   const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset());
 133   const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset());
 134   const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset());
 135 #ifndef AARCH64
 136   const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset()
 137                              + in_ByteSize(wordSize));
 138 #endif // !AARCH64
 139 
 140   Register zero = zero_register(Rtemp);
 141 
 142   switch (state) {
 143     case atos: ldr(R0_tos, oop_addr);
 144                str(zero, oop_addr);
 145                interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 146                break;
 147 
 148 #ifdef AARCH64
 149     case ltos: ldr(R0_tos, val_addr);              break;
 150 #else
 151     case ltos: ldr(R1_tos_hi, val_addr_hi);        // fall through
 152 #endif // AARCH64
 153     case btos:                                     // fall through
 154     case ztos:                                     // fall through
 155     case ctos:                                     // fall through
 156     case stos:                                     // fall through
 157     case itos: ldr_s32(R0_tos, val_addr);          break;
 158 #ifdef __SOFTFP__
 159     case dtos: ldr(R1_tos_hi, val_addr_hi);        // fall through
 160     case ftos: ldr(R0_tos, val_addr);              break;
 161 #else
 162     case ftos: ldr_float (S0_tos, val_addr);       break;
 163     case dtos: ldr_double(D0_tos, val_addr);       break;
 164 #endif // __SOFTFP__
 165     case vtos: /* nothing to do */                 break;
 166     default  : ShouldNotReachHere();
 167   }
 168   // Clean up tos value in the thread object
 169   str(zero, val_addr);
 170 #ifndef AARCH64
 171   str(zero, val_addr_hi);
 172 #endif // !AARCH64
 173 
 174   mov(Rtemp, (int) ilgl);
 175   str_32(Rtemp, tos_addr);
 176 }
 177 
 178 
 179 // Blows R2, Rtemp.
 180 void InterpreterMacroAssembler::check_and_handle_earlyret() {
 181   if (can_force_early_return()) {
 182     Label L;
 183     const Register thread_state = R2_tmp;
 184 
 185     ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
 186     cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit;
 187 
 188     // Initiate earlyret handling only if it is not already being processed.
 189     // If the flag has the earlyret_processing bit set, it means that this code
 190     // is called *during* earlyret handling - we don't want to reenter.
 191 
 192     ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset()));
 193     cmp(Rtemp, JvmtiThreadState::earlyret_pending);
 194     b(L, ne);
 195 
 196     // Call Interpreter::remove_activation_early_entry() to get the address of the
 197     // same-named entrypoint in the generated interpreter code.
 198 
 199     ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset()));
 200     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0);
 201 
 202     jump(R0);
 203 
 204     bind(L);
 205   }
 206 }
 207 
 208 
 209 // Sets reg. Blows Rtemp.
 210 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) {
 211   assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
 212   assert(reg != Rtemp, "should be different registers");
 213 
 214   ldrb(Rtemp, Address(Rbcp, bcp_offset));
 215   ldrb(reg, Address(Rbcp, bcp_offset+1));
 216   orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte));
 217 }
 218 
 219 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) {
 220   assert_different_registers(index, tmp_reg);
 221   if (index_size == sizeof(u2)) {
 222     // load bytes of index separately to avoid unaligned access
 223     ldrb(index, Address(Rbcp, bcp_offset+1));
 224     ldrb(tmp_reg, Address(Rbcp, bcp_offset));
 225     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 226   } else if (index_size == sizeof(u4)) {
 227     // TODO-AARCH64: consider using unaligned access here
 228     ldrb(index, Address(Rbcp, bcp_offset+3));
 229     ldrb(tmp_reg, Address(Rbcp, bcp_offset+2));
 230     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 231     ldrb(tmp_reg, Address(Rbcp, bcp_offset+1));
 232     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 233     ldrb(tmp_reg, Address(Rbcp, bcp_offset));
 234     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 235     // Check if the secondary index definition is still ~x, otherwise
 236     // we have to change the following assembler code to calculate the
 237     // plain index.
 238     assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");
 239     mvn_32(index, index);  // convert to plain index
 240   } else if (index_size == sizeof(u1)) {
 241     ldrb(index, Address(Rbcp, bcp_offset));
 242   } else {
 243     ShouldNotReachHere();
 244   }
 245 }
 246 
 247 // Sets cache, index.
 248 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) {
 249   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 250   assert_different_registers(cache, index);
 251 
 252   get_index_at_bcp(index, bcp_offset, cache, index_size);
 253 
 254   // load constant pool cache pointer
 255   ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
 256 
 257   // convert from field index to ConstantPoolCacheEntry index
 258   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
 259   // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called
 260   logical_shift_left(index, index, 2);
 261 }
 262 
 263 // Sets cache, index, bytecode.
 264 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) {
 265   get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);
 266   // caution index and bytecode can be the same
 267   add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord));
 268 #ifdef AARCH64
 269   add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()));
 270   ldarb(bytecode, bytecode);
 271 #else
 272   ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())));
 273   TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true);
 274 #endif // AARCH64
 275 }
 276 
 277 // Sets cache. Blows reg_tmp.
 278 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) {
 279   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 280   assert_different_registers(cache, reg_tmp);
 281 
 282   get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size);
 283 
 284   // load constant pool cache pointer
 285   ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
 286 
 287   // skip past the header
 288   add(cache, cache, in_bytes(ConstantPoolCache::base_offset()));
 289   // convert from field index to ConstantPoolCacheEntry index
 290   // and from word offset to byte offset
 291   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
 292   add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord));
 293 }
 294 
 295 // Load object from cpool->resolved_references(index)
 296 void InterpreterMacroAssembler::load_resolved_reference_at_index(
 297                                            Register result, Register index) {
 298   assert_different_registers(result, index);
 299   get_constant_pool(result);
 300 
 301   Register cache = result;
 302   // load pointer for resolved_references[] objArray
 303   ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes()));
 304   ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes()));
 305   resolve_oop_handle(cache);
 306   // Add in the index
 307   // convert from field index to resolved_references() index and from
 308   // word index to byte offset. Since this is a java object, it can be compressed
 309   add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop));
 310   load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 311 }
 312 
 313 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
 314                                            Register Rcpool, Register Rindex, Register Rklass) {
 315   add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord));
 316   ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index
 317   ldr(Rklass, Address(Rcpool,  ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses
 318   add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord));
 319   ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes()));
 320 }
 321 
 322 // Generate a subtype check: branch to not_subtype if sub_klass is
 323 // not a subtype of super_klass.
 324 // Profiling code for the subtype check failure (profile_typecheck_failed)
 325 // should be explicitly generated by the caller in the not_subtype case.
 326 // Blows Rtemp, tmp1, tmp2.
 327 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 328                                                   Register Rsuper_klass,
 329                                                   Label &not_subtype,
 330                                                   Register tmp1,
 331                                                   Register tmp2) {
 332 
 333   assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp);
 334   Label ok_is_subtype, loop, update_cache;
 335 
 336   const Register super_check_offset = tmp1;
 337   const Register cached_super = tmp2;
 338 
 339   // Profile the not-null value's klass.
 340   profile_typecheck(tmp1, Rsub_klass);
 341 
 342   // Load the super-klass's check offset into
 343   ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset()));
 344 
 345   // Check for self
 346   cmp(Rsub_klass, Rsuper_klass);
 347 
 348   // Load from the sub-klass's super-class display list, or a 1-word cache of
 349   // the secondary superclass list, or a failing value with a sentinel offset
 350   // if the super-klass is an interface or exceptionally deep in the Java
 351   // hierarchy and we have to scan the secondary superclass list the hard way.
 352   // See if we get an immediate positive hit
 353   ldr(cached_super, Address(Rsub_klass, super_check_offset));
 354 
 355   cond_cmp(Rsuper_klass, cached_super, ne);
 356   b(ok_is_subtype, eq);
 357 
 358   // Check for immediate negative hit
 359   cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset()));
 360   b(not_subtype, ne);
 361 
 362   // Now do a linear scan of the secondary super-klass chain.
 363   const Register supers_arr = tmp1;
 364   const Register supers_cnt = tmp2;
 365   const Register cur_super  = Rtemp;
 366 
 367   // Load objArrayOop of secondary supers.
 368   ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset()));
 369 
 370   ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length
 371 #ifdef AARCH64
 372   cbz(supers_cnt, not_subtype);
 373   add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes());
 374 #else
 375   cmp(supers_cnt, 0);
 376 
 377   // Skip to the start of array elements and prefetch the first super-klass.
 378   ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne);
 379   b(not_subtype, eq);
 380 #endif // AARCH64
 381 
 382   bind(loop);
 383 
 384 #ifdef AARCH64
 385   ldr(cur_super, Address(supers_arr, wordSize, post_indexed));
 386 #endif // AARCH64
 387 
 388   cmp(cur_super, Rsuper_klass);
 389   b(update_cache, eq);
 390 
 391   subs(supers_cnt, supers_cnt, 1);
 392 
 393 #ifndef AARCH64
 394   ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne);
 395 #endif // !AARCH64
 396 
 397   b(loop, ne);
 398 
 399   b(not_subtype);
 400 
 401   bind(update_cache);
 402   // Must be equal but missed in cache.  Update cache.
 403   str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset()));
 404 
 405   bind(ok_is_subtype);
 406 }
 407 
 408 
 409 // The 1st part of the store check.
 410 // Sets card_table_base register.
 411 void InterpreterMacroAssembler::store_check_part1(Register card_table_base) {
 412   // Check barrier set type (should be card table) and element size
 413   BarrierSet* bs = Universe::heap()->barrier_set();
 414   assert(bs->kind() == BarrierSet::CardTableBarrierSet,
 415          "Wrong barrier set kind");
 416 
 417   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
 418   CardTable* ct = ctbs->card_table();
 419   assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "Adjust store check code");
 420 
 421   // Load card table base address.
 422 
 423   /* Performance note.
 424 
 425      There is an alternative way of loading card table base address
 426      from thread descriptor, which may look more efficient:
 427 
 428      ldr(card_table_base, Address(Rthread, JavaThread::card_table_base_offset()));
 429 
 430      However, performance measurements of micro benchmarks and specJVM98
 431      showed that loading of card table base from thread descriptor is
 432      7-18% slower compared to loading of literal embedded into the code.
 433      Possible cause is a cache miss (card table base address resides in a
 434      rarely accessed area of thread descriptor).
 435   */
 436   // TODO-AARCH64 Investigate if mov_slow is faster than ldr from Rthread on AArch64
 437   mov_address(card_table_base, (address)ct->byte_map_base(), symbolic_Relocation::card_table_reference);
 438 }
 439 
 440 // The 2nd part of the store check.
 441 void InterpreterMacroAssembler::store_check_part2(Register obj, Register card_table_base, Register tmp) {
 442   assert_different_registers(obj, card_table_base, tmp);
 443 
 444   assert(CardTable::dirty_card_val() == 0, "Dirty card value must be 0 due to optimizations.");
 445 #ifdef AARCH64
 446   add(card_table_base, card_table_base, AsmOperand(obj, lsr, CardTable::card_shift));
 447   Address card_table_addr(card_table_base);
 448 #else
 449   Address card_table_addr(card_table_base, obj, lsr, CardTable::card_shift);
 450 #endif
 451 
 452   if (UseCondCardMark) {
 453     if (UseConcMarkSweepGC) {
 454       membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), noreg);
 455     }
 456     Label already_dirty;
 457 
 458     ldrb(tmp, card_table_addr);
 459     cbz(tmp, already_dirty);
 460 
 461     set_card(card_table_base, card_table_addr, tmp);
 462     bind(already_dirty);
 463 
 464   } else {
 465     if (UseConcMarkSweepGC && CMSPrecleaningEnabled) {
 466       membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg);
 467     }
 468     set_card(card_table_base, card_table_addr, tmp);
 469   }
 470 }
 471 
 472 void InterpreterMacroAssembler::set_card(Register card_table_base, Address card_table_addr, Register tmp) {
 473 #ifdef AARCH64
 474   strb(ZR, card_table_addr);
 475 #else
 476   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(Universe::heap()->barrier_set());
 477   CardTable* ct = ctbs->card_table();
 478   if ((((uintptr_t)ct->byte_map_base() & 0xff) == 0)) {
 479     // Card table is aligned so the lowest byte of the table address base is zero.
 480     // This works only if the code is not saved for later use, possibly
 481     // in a context where the base would no longer be aligned.
 482     strb(card_table_base, card_table_addr);
 483   } else {
 484     mov(tmp, 0);
 485     strb(tmp, card_table_addr);
 486   }
 487 #endif // AARCH64
 488 }
 489 
 490 //////////////////////////////////////////////////////////////////////////////////
 491 
 492 
 493 // Java Expression Stack
 494 
 495 void InterpreterMacroAssembler::pop_ptr(Register r) {
 496   assert(r != Rstack_top, "unpredictable instruction");
 497   ldr(r, Address(Rstack_top, wordSize, post_indexed));
 498 }
 499 
 500 void InterpreterMacroAssembler::pop_i(Register r) {
 501   assert(r != Rstack_top, "unpredictable instruction");
 502   ldr_s32(r, Address(Rstack_top, wordSize, post_indexed));
 503   zap_high_non_significant_bits(r);
 504 }
 505 
 506 #ifdef AARCH64
 507 void InterpreterMacroAssembler::pop_l(Register r) {
 508   assert(r != Rstack_top, "unpredictable instruction");
 509   ldr(r, Address(Rstack_top, 2*wordSize, post_indexed));
 510 }
 511 #else
 512 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) {
 513   assert_different_registers(lo, hi);
 514   assert(lo < hi, "lo must be < hi");
 515   pop(RegisterSet(lo) | RegisterSet(hi));
 516 }
 517 #endif // AARCH64
 518 
 519 void InterpreterMacroAssembler::pop_f(FloatRegister fd) {
 520 #ifdef AARCH64
 521   ldr_s(fd, Address(Rstack_top, wordSize, post_indexed));
 522 #else
 523   fpops(fd);
 524 #endif // AARCH64
 525 }
 526 
 527 void InterpreterMacroAssembler::pop_d(FloatRegister fd) {
 528 #ifdef AARCH64
 529   ldr_d(fd, Address(Rstack_top, 2*wordSize, post_indexed));
 530 #else
 531   fpopd(fd);
 532 #endif // AARCH64
 533 }
 534 
 535 
 536 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value.
 537 void InterpreterMacroAssembler::pop(TosState state) {
 538   switch (state) {
 539     case atos: pop_ptr(R0_tos);                              break;
 540     case btos:                                               // fall through
 541     case ztos:                                               // fall through
 542     case ctos:                                               // fall through
 543     case stos:                                               // fall through
 544     case itos: pop_i(R0_tos);                                break;
 545 #ifdef AARCH64
 546     case ltos: pop_l(R0_tos);                                break;
 547 #else
 548     case ltos: pop_l(R0_tos_lo, R1_tos_hi);                  break;
 549 #endif // AARCH64
 550 #ifdef __SOFTFP__
 551     case ftos: pop_i(R0_tos);                                break;
 552     case dtos: pop_l(R0_tos_lo, R1_tos_hi);                  break;
 553 #else
 554     case ftos: pop_f(S0_tos);                                break;
 555     case dtos: pop_d(D0_tos);                                break;
 556 #endif // __SOFTFP__
 557     case vtos: /* nothing to do */                           break;
 558     default  : ShouldNotReachHere();
 559   }
 560   interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 561 }
 562 
 563 void InterpreterMacroAssembler::push_ptr(Register r) {
 564   assert(r != Rstack_top, "unpredictable instruction");
 565   str(r, Address(Rstack_top, -wordSize, pre_indexed));
 566   check_stack_top_on_expansion();
 567 }
 568 
 569 void InterpreterMacroAssembler::push_i(Register r) {
 570   assert(r != Rstack_top, "unpredictable instruction");
 571   str_32(r, Address(Rstack_top, -wordSize, pre_indexed));
 572   check_stack_top_on_expansion();
 573 }
 574 
 575 #ifdef AARCH64
 576 void InterpreterMacroAssembler::push_l(Register r) {
 577   assert(r != Rstack_top, "unpredictable instruction");
 578   stp(r, ZR, Address(Rstack_top, -2*wordSize, pre_indexed));
 579   check_stack_top_on_expansion();
 580 }
 581 #else
 582 void InterpreterMacroAssembler::push_l(Register lo, Register hi) {
 583   assert_different_registers(lo, hi);
 584   assert(lo < hi, "lo must be < hi");
 585   push(RegisterSet(lo) | RegisterSet(hi));
 586 }
 587 #endif // AARCH64
 588 
 589 void InterpreterMacroAssembler::push_f() {
 590 #ifdef AARCH64
 591   str_s(S0_tos, Address(Rstack_top, -wordSize, pre_indexed));
 592   check_stack_top_on_expansion();
 593 #else
 594   fpushs(S0_tos);
 595 #endif // AARCH64
 596 }
 597 
 598 void InterpreterMacroAssembler::push_d() {
 599 #ifdef AARCH64
 600   str_d(D0_tos, Address(Rstack_top, -2*wordSize, pre_indexed));
 601   check_stack_top_on_expansion();
 602 #else
 603   fpushd(D0_tos);
 604 #endif // AARCH64
 605 }
 606 
 607 // Transition state -> vtos. Blows Rtemp.
 608 void InterpreterMacroAssembler::push(TosState state) {
 609   interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 610   switch (state) {
 611     case atos: push_ptr(R0_tos);                              break;
 612     case btos:                                                // fall through
 613     case ztos:                                                // fall through
 614     case ctos:                                                // fall through
 615     case stos:                                                // fall through
 616     case itos: push_i(R0_tos);                                break;
 617 #ifdef AARCH64
 618     case ltos: push_l(R0_tos);                                break;
 619 #else
 620     case ltos: push_l(R0_tos_lo, R1_tos_hi);                  break;
 621 #endif // AARCH64
 622 #ifdef __SOFTFP__
 623     case ftos: push_i(R0_tos);                                break;
 624     case dtos: push_l(R0_tos_lo, R1_tos_hi);                  break;
 625 #else
 626     case ftos: push_f();                                      break;
 627     case dtos: push_d();                                      break;
 628 #endif // __SOFTFP__
 629     case vtos: /* nothing to do */                            break;
 630     default  : ShouldNotReachHere();
 631   }
 632 }
 633 
 634 
 635 #ifndef AARCH64
 636 
 637 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value.
 638 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) {
 639 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 640   // According to interpreter calling conventions, result is returned in R0/R1,
 641   // but templates expect ftos in S0, and dtos in D0.
 642   if (state == ftos) {
 643     fmsr(S0_tos, R0);
 644   } else if (state == dtos) {
 645     fmdrr(D0_tos, R0, R1);
 646   }
 647 #endif // !__SOFTFP__ && !__ABI_HARD__
 648 }
 649 
 650 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions).
 651 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) {
 652 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 653   // According to interpreter calling conventions, result is returned in R0/R1,
 654   // so ftos (S0) and dtos (D0) are moved to R0/R1.
 655   if (state == ftos) {
 656     fmrs(R0, S0_tos);
 657   } else if (state == dtos) {
 658     fmrrd(R0, R1, D0_tos);
 659   }
 660 #endif // !__SOFTFP__ && !__ABI_HARD__
 661 }
 662 
 663 #endif // !AARCH64
 664 
 665 
 666 // Helpers for swap and dup
 667 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
 668   ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
 669 }
 670 
 671 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
 672   str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
 673 }
 674 
 675 
 676 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
 677 #ifdef AARCH64
 678   check_no_cached_stack_top(Rtemp);
 679   save_stack_top();
 680   cut_sp_before_call();
 681   mov(Rparams, Rstack_top);
 682 #endif // AARCH64
 683 
 684   // set sender sp
 685   mov(Rsender_sp, SP);
 686 
 687 #ifndef AARCH64
 688   // record last_sp
 689   str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
 690 #endif // !AARCH64
 691 }
 692 
 693 // Jump to from_interpreted entry of a call unless single stepping is possible
 694 // in this thread in which case we must call the i2i entry
 695 void InterpreterMacroAssembler::jump_from_interpreted(Register method) {
 696   assert_different_registers(method, Rtemp);
 697 
 698   prepare_to_jump_from_interpreted();
 699 
 700   if (can_post_interpreter_events()) {
 701     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
 702     // compiled code in threads for which the event is enabled.  Check here for
 703     // interp_only_mode if these events CAN be enabled.
 704 
 705     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
 706 #ifdef AARCH64
 707     {
 708       Label not_interp_only_mode;
 709 
 710       cbz(Rtemp, not_interp_only_mode);
 711       indirect_jump(Address(method, Method::interpreter_entry_offset()), Rtemp);
 712 
 713       bind(not_interp_only_mode);
 714     }
 715 #else
 716     cmp(Rtemp, 0);
 717     ldr(PC, Address(method, Method::interpreter_entry_offset()), ne);
 718 #endif // AARCH64
 719   }
 720 
 721   indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp);
 722 }
 723 
 724 
 725 void InterpreterMacroAssembler::restore_dispatch() {
 726   mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos));
 727 }
 728 
 729 
 730 // The following two routines provide a hook so that an implementation
 731 // can schedule the dispatch in two parts.
 732 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
 733   // Nothing ARM-specific to be done here.
 734 }
 735 
 736 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
 737   dispatch_next(state, step);
 738 }
 739 
 740 void InterpreterMacroAssembler::dispatch_base(TosState state,
 741                                               DispatchTableMode table_mode,
 742                                               bool verifyoop) {
 743   if (VerifyActivationFrameSize) {
 744     Label L;
 745 #ifdef AARCH64
 746     mov(Rtemp, SP);
 747     sub(Rtemp, FP, Rtemp);
 748 #else
 749     sub(Rtemp, FP, SP);
 750 #endif // AARCH64
 751     int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
 752     cmp(Rtemp, min_frame_size);
 753     b(L, ge);
 754     stop("broken stack frame");
 755     bind(L);
 756   }
 757 
 758   if (verifyoop) {
 759     interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 760   }
 761 
 762   if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) {
 763     zap_high_non_significant_bits(R0_tos);
 764   }
 765 
 766 #ifdef ASSERT
 767   Label L;
 768   mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos));
 769   cmp(Rtemp, RdispatchTable);
 770   b(L, eq);
 771   stop("invalid RdispatchTable");
 772   bind(L);
 773 #endif
 774 
 775   if (table_mode == DispatchDefault) {
 776     if (state == vtos) {
 777       indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp);
 778     } else {
 779 #ifdef AARCH64
 780       sub(Rtemp, R3_bytecode, (Interpreter::distance_from_dispatch_table(vtos) -
 781                            Interpreter::distance_from_dispatch_table(state)));
 782       indirect_jump(Address::indexed_ptr(RdispatchTable, Rtemp), Rtemp);
 783 #else
 784       // on 32-bit ARM this method is faster than the one above.
 785       sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) -
 786                            Interpreter::distance_from_dispatch_table(state)) * wordSize);
 787       indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 788 #endif
 789     }
 790   } else {
 791     assert(table_mode == DispatchNormal, "invalid dispatch table mode");
 792     address table = (address) Interpreter::normal_table(state);
 793     mov_slow(Rtemp, table);
 794     indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 795   }
 796 
 797   nop(); // to avoid filling CPU pipeline with invalid instructions
 798   nop();
 799 }
 800 
 801 void InterpreterMacroAssembler::dispatch_only(TosState state) {
 802   dispatch_base(state, DispatchDefault);
 803 }
 804 
 805 
 806 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
 807   dispatch_base(state, DispatchNormal);
 808 }
 809 
 810 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
 811   dispatch_base(state, DispatchNormal, false);
 812 }
 813 
 814 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) {
 815   // load next bytecode and advance Rbcp
 816   ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed));
 817   dispatch_base(state, DispatchDefault);
 818 }
 819 
 820 void InterpreterMacroAssembler::narrow(Register result) {
 821   // mask integer result to narrower return type.
 822   const Register Rtmp = R2;
 823 
 824   // get method type
 825   ldr(Rtmp, Address(Rmethod, Method::const_offset()));
 826   ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset()));
 827 
 828   Label notBool, notByte, notChar, done;
 829   cmp(Rtmp, T_INT);
 830   b(done, eq);
 831 
 832   cmp(Rtmp, T_BOOLEAN);
 833   b(notBool, ne);
 834   and_32(result, result, 1);
 835   b(done);
 836 
 837   bind(notBool);
 838   cmp(Rtmp, T_BYTE);
 839   b(notByte, ne);
 840   sign_extend(result, result, 8);
 841   b(done);
 842 
 843   bind(notByte);
 844   cmp(Rtmp, T_CHAR);
 845   b(notChar, ne);
 846   zero_extend(result, result, 16);
 847   b(done);
 848 
 849   bind(notChar);
 850   // cmp(Rtmp, T_SHORT);
 851   // b(done, ne);
 852   sign_extend(result, result, 16);
 853 
 854   // Nothing to do
 855   bind(done);
 856 }
 857 
 858 // remove activation
 859 //
 860 // Unlock the receiver if this is a synchronized method.
 861 // Unlock any Java monitors from syncronized blocks.
 862 // Remove the activation from the stack.
 863 //
 864 // If there are locked Java monitors
 865 //    If throw_monitor_exception
 866 //       throws IllegalMonitorStateException
 867 //    Else if install_monitor_exception
 868 //       installs IllegalMonitorStateException
 869 //    Else
 870 //       no error processing
 871 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr,
 872                                                   bool throw_monitor_exception,
 873                                                   bool install_monitor_exception,
 874                                                   bool notify_jvmdi) {
 875   Label unlock, unlocked, no_unlock;
 876 
 877   // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result.
 878 
 879   const Address do_not_unlock_if_synchronized(Rthread,
 880                          JavaThread::do_not_unlock_if_synchronized_offset());
 881 
 882   const Register Rflag = R2;
 883   const Register Raccess_flags = R3;
 884 
 885   restore_method();
 886 
 887   ldrb(Rflag, do_not_unlock_if_synchronized);
 888 
 889   // get method access flags
 890   ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset()));
 891 
 892   strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag
 893 
 894   // check if method is synchronized
 895 
 896   tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked);
 897 
 898   // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set.
 899   cbnz(Rflag, no_unlock);
 900 
 901   // unlock monitor
 902   push(state);                                   // save result
 903 
 904   // BasicObjectLock will be first in list, since this is a synchronized method. However, need
 905   // to check that the object has not been unlocked by an explicit monitorexit bytecode.
 906 
 907   const Register Rmonitor = R1;                  // fixed in unlock_object()
 908   const Register Robj = R2;
 909 
 910   // address of first monitor
 911   sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock));
 912 
 913   ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes()));
 914   cbnz(Robj, unlock);
 915 
 916   pop(state);
 917 
 918   if (throw_monitor_exception) {
 919     // Entry already unlocked, need to throw exception
 920     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
 921     should_not_reach_here();
 922   } else {
 923     // Monitor already unlocked during a stack unroll.
 924     // If requested, install an illegal_monitor_state_exception.
 925     // Continue with stack unrolling.
 926     if (install_monitor_exception) {
 927       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
 928     }
 929     b(unlocked);
 930   }
 931 
 932 
 933   // Exception case for the check that all monitors are unlocked.
 934   const Register Rcur = R2;
 935   Label restart_check_monitors_unlocked, exception_monitor_is_still_locked;
 936 
 937   bind(exception_monitor_is_still_locked);
 938   // Monitor entry is still locked, need to throw exception.
 939   // Rcur: monitor entry.
 940 
 941   if (throw_monitor_exception) {
 942     // Throw exception
 943     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
 944     should_not_reach_here();
 945   } else {
 946     // Stack unrolling. Unlock object and install illegal_monitor_exception
 947     // Unlock does not block, so don't have to worry about the frame
 948 
 949     push(state);
 950     mov(R1, Rcur);
 951     unlock_object(R1);
 952 
 953     if (install_monitor_exception) {
 954       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
 955     }
 956 
 957     pop(state);
 958     b(restart_check_monitors_unlocked);
 959   }
 960 
 961   bind(unlock);
 962   unlock_object(Rmonitor);
 963   pop(state);
 964 
 965   // Check that for block-structured locking (i.e., that all locked objects has been unlocked)
 966   bind(unlocked);
 967 
 968   // Check that all monitors are unlocked
 969   {
 970     Label loop;
 971 
 972     const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 973     const Register Rbottom = R3;
 974     const Register Rcur_obj = Rtemp;
 975 
 976     bind(restart_check_monitors_unlocked);
 977 
 978     ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
 979                                  // points to current entry, starting with top-most entry
 980     sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize);
 981                                  // points to word before bottom of monitor block
 982 
 983     cmp(Rcur, Rbottom);          // check if there are no monitors
 984 #ifndef AARCH64
 985     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
 986                                  // prefetch monitor's object
 987 #endif // !AARCH64
 988     b(no_unlock, eq);
 989 
 990     bind(loop);
 991 #ifdef AARCH64
 992     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()));
 993 #endif // AARCH64
 994     // check if current entry is used
 995     cbnz(Rcur_obj, exception_monitor_is_still_locked);
 996 
 997     add(Rcur, Rcur, entry_size);      // otherwise advance to next entry
 998     cmp(Rcur, Rbottom);               // check if bottom reached
 999 #ifndef AARCH64
1000     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
1001                                       // prefetch monitor's object
1002 #endif // !AARCH64
1003     b(loop, ne);                      // if not at bottom then check this entry
1004   }
1005 
1006   bind(no_unlock);
1007 
1008   // jvmti support
1009   if (notify_jvmdi) {
1010     notify_method_exit(state, NotifyJVMTI);     // preserve TOSCA
1011   } else {
1012     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
1013   }
1014 
1015   // remove activation
1016 #ifdef AARCH64
1017   ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
1018   ldp(FP, LR, Address(FP));
1019   mov(SP, Rtemp);
1020 #else
1021   mov(Rtemp, FP);
1022   ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
1023   ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
1024 #endif
1025 
1026   if (ret_addr != LR) {
1027     mov(ret_addr, LR);
1028   }
1029 }
1030 
1031 
1032 // At certain points in the method invocation the monitor of
1033 // synchronized methods hasn't been entered yet.
1034 // To correctly handle exceptions at these points, we set the thread local
1035 // variable _do_not_unlock_if_synchronized to true. The remove_activation will
1036 // check this flag.
1037 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) {
1038   const Address do_not_unlock_if_synchronized(Rthread,
1039                          JavaThread::do_not_unlock_if_synchronized_offset());
1040   if (flag) {
1041     mov(tmp, 1);
1042     strb(tmp, do_not_unlock_if_synchronized);
1043   } else {
1044     strb(zero_register(tmp), do_not_unlock_if_synchronized);
1045   }
1046 }
1047 
1048 // Lock object
1049 //
1050 // Argument: R1 : Points to BasicObjectLock to be used for locking.
1051 // Must be initialized with object to lock.
1052 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM.
1053 void InterpreterMacroAssembler::lock_object(Register Rlock) {
1054   assert(Rlock == R1, "the second argument");
1055 
1056   if (UseHeavyMonitors) {
1057     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
1058   } else {
1059     Label done;
1060 
1061     const Register Robj = R2;
1062     const Register Rmark = R3;
1063     assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp);
1064 
1065     const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1066     const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1067     const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
1068 
1069     Label already_locked, slow_case;
1070 
1071     // Load object pointer
1072     ldr(Robj, Address(Rlock, obj_offset));
1073 
1074     if (UseBiasedLocking) {
1075       biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case);
1076     }
1077 
1078 #ifdef AARCH64
1079     assert(oopDesc::mark_offset_in_bytes() == 0, "must be");
1080     ldr(Rmark, Robj);
1081 
1082     // Test if object is already locked
1083     assert(markOopDesc::unlocked_value == 1, "adjust this code");
1084     tbz(Rmark, exact_log2(markOopDesc::unlocked_value), already_locked);
1085 
1086 #else // AARCH64
1087 
1088     // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
1089     // That would be acceptable as ether CAS or slow case path is taken in that case.
1090     // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as
1091     // loads are satisfied from a store queue if performed on the same processor).
1092 
1093     assert(oopDesc::mark_offset_in_bytes() == 0, "must be");
1094     ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes()));
1095 
1096     // Test if object is already locked
1097     tst(Rmark, markOopDesc::unlocked_value);
1098     b(already_locked, eq);
1099 
1100 #endif // !AARCH64
1101     // Save old object->mark() into BasicLock's displaced header
1102     str(Rmark, Address(Rlock, mark_offset));
1103 
1104     cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case);
1105 
1106 #ifndef PRODUCT
1107     if (PrintBiasedLockingStatistics) {
1108       cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr());
1109     }
1110 #endif //!PRODUCT
1111 
1112     b(done);
1113 
1114     // If we got here that means the object is locked by ether calling thread or another thread.
1115     bind(already_locked);
1116     // Handling of locked objects: recursive locks and slow case.
1117 
1118     // Fast check for recursive lock.
1119     //
1120     // Can apply the optimization only if this is a stack lock
1121     // allocated in this thread. For efficiency, we can focus on
1122     // recently allocated stack locks (instead of reading the stack
1123     // base and checking whether 'mark' points inside the current
1124     // thread stack):
1125     //  1) (mark & 3) == 0
1126     //  2) SP <= mark < SP + os::pagesize()
1127     //
1128     // Warning: SP + os::pagesize can overflow the stack base. We must
1129     // neither apply the optimization for an inflated lock allocated
1130     // just above the thread stack (this is why condition 1 matters)
1131     // nor apply the optimization if the stack lock is inside the stack
1132     // of another thread. The latter is avoided even in case of overflow
1133     // because we have guard pages at the end of all stacks. Hence, if
1134     // we go over the stack base and hit the stack of another thread,
1135     // this should not be in a writeable area that could contain a
1136     // stack lock allocated by that thread. As a consequence, a stack
1137     // lock less than page size away from SP is guaranteed to be
1138     // owned by the current thread.
1139     //
1140     // Note: assuming SP is aligned, we can check the low bits of
1141     // (mark-SP) instead of the low bits of mark. In that case,
1142     // assuming page size is a power of 2, we can merge the two
1143     // conditions into a single test:
1144     // => ((mark - SP) & (3 - os::pagesize())) == 0
1145 
1146 #ifdef AARCH64
1147     // Use the single check since the immediate is OK for AARCH64
1148     sub(R0, Rmark, Rstack_top);
1149     intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size());
1150     Assembler::LogicalImmediate imm(mask, false);
1151     ands(R0, R0, imm);
1152 
1153     // For recursive case store 0 into lock record.
1154     // It is harmless to store it unconditionally as lock record contains some garbage
1155     // value in its _displaced_header field by this moment.
1156     str(ZR, Address(Rlock, mark_offset));
1157 
1158 #else // AARCH64
1159     // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand.
1160     // Check independently the low bits and the distance to SP.
1161     // -1- test low 2 bits
1162     movs(R0, AsmOperand(Rmark, lsl, 30));
1163     // -2- test (mark - SP) if the low two bits are 0
1164     sub(R0, Rmark, SP, eq);
1165     movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq);
1166     // If still 'eq' then recursive locking OK: store 0 into lock record
1167     str(R0, Address(Rlock, mark_offset), eq);
1168 
1169 #endif // AARCH64
1170 
1171 #ifndef PRODUCT
1172     if (PrintBiasedLockingStatistics) {
1173       cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr());
1174     }
1175 #endif // !PRODUCT
1176 
1177     b(done, eq);
1178 
1179     bind(slow_case);
1180 
1181     // Call the runtime routine for slow case
1182     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
1183 
1184     bind(done);
1185   }
1186 }
1187 
1188 
1189 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
1190 //
1191 // Argument: R1: Points to BasicObjectLock structure for lock
1192 // Throw an IllegalMonitorException if object is not locked by current thread
1193 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM.
1194 void InterpreterMacroAssembler::unlock_object(Register Rlock) {
1195   assert(Rlock == R1, "the second argument");
1196 
1197   if (UseHeavyMonitors) {
1198     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
1199   } else {
1200     Label done, slow_case;
1201 
1202     const Register Robj = R2;
1203     const Register Rmark = R3;
1204     const Register Rresult = R0;
1205     assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp);
1206 
1207     const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1208     const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1209     const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
1210 
1211     const Register Rzero = zero_register(Rtemp);
1212 
1213     // Load oop into Robj
1214     ldr(Robj, Address(Rlock, obj_offset));
1215 
1216     // Free entry
1217     str(Rzero, Address(Rlock, obj_offset));
1218 
1219     if (UseBiasedLocking) {
1220       biased_locking_exit(Robj, Rmark, done);
1221     }
1222 
1223     // Load the old header from BasicLock structure
1224     ldr(Rmark, Address(Rlock, mark_offset));
1225 
1226     // Test for recursion (zero mark in BasicLock)
1227     cbz(Rmark, done);
1228 
1229     bool allow_fallthrough_on_failure = true;
1230 
1231     cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure);
1232 
1233     b(done, eq);
1234 
1235     bind(slow_case);
1236 
1237     // Call the runtime routine for slow case.
1238     str(Robj, Address(Rlock, obj_offset)); // restore obj
1239     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
1240 
1241     bind(done);
1242   }
1243 }
1244 
1245 
1246 // Test ImethodDataPtr.  If it is null, continue at the specified label
1247 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) {
1248   assert(ProfileInterpreter, "must be profiling interpreter");
1249   ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1250   cbz(mdp, zero_continue);
1251 }
1252 
1253 
1254 // Set the method data pointer for the current bcp.
1255 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR.
1256 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
1257   assert(ProfileInterpreter, "must be profiling interpreter");
1258   Label set_mdp;
1259 
1260   // Test MDO to avoid the call if it is NULL.
1261   ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
1262   cbz(Rtemp, set_mdp);
1263 
1264   mov(R0, Rmethod);
1265   mov(R1, Rbcp);
1266   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1);
1267   // R0/W0: mdi
1268 
1269   // mdo is guaranteed to be non-zero here, we checked for it before the call.
1270   ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
1271   add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()));
1272   add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0);
1273 
1274   bind(set_mdp);
1275   str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1276 }
1277 
1278 
1279 void InterpreterMacroAssembler::verify_method_data_pointer() {
1280   assert(ProfileInterpreter, "must be profiling interpreter");
1281 #ifdef ASSERT
1282   Label verify_continue;
1283   save_caller_save_registers();
1284 
1285   const Register Rmdp = R2;
1286   test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue
1287 
1288   // If the mdp is valid, it will point to a DataLayout header which is
1289   // consistent with the bcp.  The converse is highly probable also.
1290 
1291   ldrh(R3, Address(Rmdp, DataLayout::bci_offset()));
1292   ldr(Rtemp, Address(Rmethod, Method::const_offset()));
1293   add(R3, R3, Rtemp);
1294   add(R3, R3, in_bytes(ConstMethod::codes_offset()));
1295   cmp(R3, Rbcp);
1296   b(verify_continue, eq);
1297 
1298   mov(R0, Rmethod);
1299   mov(R1, Rbcp);
1300   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp);
1301 
1302   bind(verify_continue);
1303   restore_caller_save_registers();
1304 #endif // ASSERT
1305 }
1306 
1307 
1308 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) {
1309   assert(ProfileInterpreter, "must be profiling interpreter");
1310   assert_different_registers(mdp_in, value);
1311   str(value, Address(mdp_in, offset));
1312 }
1313 
1314 
1315 // Increments mdp data. Sets bumped_count register to adjusted counter.
1316 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
1317                                                       int offset,
1318                                                       Register bumped_count,
1319                                                       bool decrement) {
1320   assert(ProfileInterpreter, "must be profiling interpreter");
1321 
1322   // Counter address
1323   Address data(mdp_in, offset);
1324   assert_different_registers(mdp_in, bumped_count);
1325 
1326   increment_mdp_data_at(data, bumped_count, decrement);
1327 }
1328 
1329 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) {
1330   assert_different_registers(mdp_in, Rtemp);
1331   assert(ProfileInterpreter, "must be profiling interpreter");
1332   assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range");
1333 
1334   // Set the flag
1335   ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
1336   orr(Rtemp, Rtemp, (unsigned)flag_byte_constant);
1337   strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
1338 }
1339 
1340 
1341 // Increments mdp data. Sets bumped_count register to adjusted counter.
1342 void InterpreterMacroAssembler::increment_mdp_data_at(Address data,
1343                                                       Register bumped_count,
1344                                                       bool decrement) {
1345   assert(ProfileInterpreter, "must be profiling interpreter");
1346 
1347   ldr(bumped_count, data);
1348   if (decrement) {
1349     // Decrement the register. Set condition codes.
1350     subs(bumped_count, bumped_count, DataLayout::counter_increment);
1351     // Avoid overflow.
1352 #ifdef AARCH64
1353     assert(DataLayout::counter_increment == 1, "required for cinc");
1354     cinc(bumped_count, bumped_count, pl);
1355 #else
1356     add(bumped_count, bumped_count, DataLayout::counter_increment, pl);
1357 #endif // AARCH64
1358   } else {
1359     // Increment the register. Set condition codes.
1360     adds(bumped_count, bumped_count, DataLayout::counter_increment);
1361     // Avoid overflow.
1362 #ifdef AARCH64
1363     assert(DataLayout::counter_increment == 1, "required for cinv");
1364     cinv(bumped_count, bumped_count, mi); // inverts 0x80..00 back to 0x7f..ff
1365 #else
1366     sub(bumped_count, bumped_count, DataLayout::counter_increment, mi);
1367 #endif // AARCH64
1368   }
1369   str(bumped_count, data);
1370 }
1371 
1372 
1373 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
1374                                                  int offset,
1375                                                  Register value,
1376                                                  Register test_value_out,
1377                                                  Label& not_equal_continue) {
1378   assert(ProfileInterpreter, "must be profiling interpreter");
1379   assert_different_registers(mdp_in, test_value_out, value);
1380 
1381   ldr(test_value_out, Address(mdp_in, offset));
1382   cmp(test_value_out, value);
1383 
1384   b(not_equal_continue, ne);
1385 }
1386 
1387 
1388 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) {
1389   assert(ProfileInterpreter, "must be profiling interpreter");
1390   assert_different_registers(mdp_in, reg_temp);
1391 
1392   ldr(reg_temp, Address(mdp_in, offset_of_disp));
1393   add(mdp_in, mdp_in, reg_temp);
1394   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1395 }
1396 
1397 
1398 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) {
1399   assert(ProfileInterpreter, "must be profiling interpreter");
1400   assert_different_registers(mdp_in, reg_offset, reg_tmp);
1401 
1402   ldr(reg_tmp, Address(mdp_in, reg_offset));
1403   add(mdp_in, mdp_in, reg_tmp);
1404   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1405 }
1406 
1407 
1408 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) {
1409   assert(ProfileInterpreter, "must be profiling interpreter");
1410   add(mdp_in, mdp_in, constant);
1411   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1412 }
1413 
1414 
1415 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
1416 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
1417   assert(ProfileInterpreter, "must be profiling interpreter");
1418   assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp);
1419 
1420   mov(R1, return_bci);
1421   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1);
1422 }
1423 
1424 
1425 // Sets mdp, bumped_count registers, blows Rtemp.
1426 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) {
1427   assert_different_registers(mdp, bumped_count);
1428 
1429   if (ProfileInterpreter) {
1430     Label profile_continue;
1431 
1432     // If no method data exists, go to profile_continue.
1433     // Otherwise, assign to mdp
1434     test_method_data_pointer(mdp, profile_continue);
1435 
1436     // We are taking a branch. Increment the taken count.
1437     increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count);
1438 
1439     // The method data pointer needs to be updated to reflect the new target.
1440     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp);
1441 
1442     bind (profile_continue);
1443   }
1444 }
1445 
1446 
1447 // Sets mdp, blows Rtemp.
1448 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
1449   assert_different_registers(mdp, Rtemp);
1450 
1451   if (ProfileInterpreter) {
1452     Label profile_continue;
1453 
1454     // If no method data exists, go to profile_continue.
1455     test_method_data_pointer(mdp, profile_continue);
1456 
1457     // We are taking a branch.  Increment the not taken count.
1458     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp);
1459 
1460     // The method data pointer needs to be updated to correspond to the next bytecode
1461     update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
1462 
1463     bind (profile_continue);
1464   }
1465 }
1466 
1467 
1468 // Sets mdp, blows Rtemp.
1469 void InterpreterMacroAssembler::profile_call(Register mdp) {
1470   assert_different_registers(mdp, Rtemp);
1471 
1472   if (ProfileInterpreter) {
1473     Label profile_continue;
1474 
1475     // If no method data exists, go to profile_continue.
1476     test_method_data_pointer(mdp, profile_continue);
1477 
1478     // We are making a call.  Increment the count.
1479     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1480 
1481     // The method data pointer needs to be updated to reflect the new target.
1482     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1483 
1484     bind (profile_continue);
1485   }
1486 }
1487 
1488 
1489 // Sets mdp, blows Rtemp.
1490 void InterpreterMacroAssembler::profile_final_call(Register mdp) {
1491   if (ProfileInterpreter) {
1492     Label profile_continue;
1493 
1494     // If no method data exists, go to profile_continue.
1495     test_method_data_pointer(mdp, profile_continue);
1496 
1497     // We are making a call.  Increment the count.
1498     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1499 
1500     // The method data pointer needs to be updated to reflect the new target.
1501     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1502 
1503     bind (profile_continue);
1504   }
1505 }
1506 
1507 
1508 // Sets mdp, blows Rtemp.
1509 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) {
1510   assert_different_registers(mdp, receiver, Rtemp);
1511 
1512   if (ProfileInterpreter) {
1513     Label profile_continue;
1514 
1515     // If no method data exists, go to profile_continue.
1516     test_method_data_pointer(mdp, profile_continue);
1517 
1518     Label skip_receiver_profile;
1519     if (receiver_can_be_null) {
1520       Label not_null;
1521       cbnz(receiver, not_null);
1522       // We are making a call.  Increment the count for null receiver.
1523       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1524       b(skip_receiver_profile);
1525       bind(not_null);
1526     }
1527 
1528     // Record the receiver type.
1529     record_klass_in_profile(receiver, mdp, Rtemp, true);
1530     bind(skip_receiver_profile);
1531 
1532     // The method data pointer needs to be updated to reflect the new target.
1533     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1534     bind(profile_continue);
1535   }
1536 }
1537 
1538 
1539 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1540                                         Register receiver, Register mdp,
1541                                         Register reg_tmp,
1542                                         int start_row, Label& done, bool is_virtual_call) {
1543   if (TypeProfileWidth == 0)
1544     return;
1545 
1546   assert_different_registers(receiver, mdp, reg_tmp);
1547 
1548   int last_row = VirtualCallData::row_limit() - 1;
1549   assert(start_row <= last_row, "must be work left to do");
1550   // Test this row for both the receiver and for null.
1551   // Take any of three different outcomes:
1552   //   1. found receiver => increment count and goto done
1553   //   2. found null => keep looking for case 1, maybe allocate this cell
1554   //   3. found something else => keep looking for cases 1 and 2
1555   // Case 3 is handled by a recursive call.
1556   for (int row = start_row; row <= last_row; row++) {
1557     Label next_test;
1558 
1559     // See if the receiver is receiver[n].
1560     int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row));
1561 
1562     test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test);
1563 
1564     // The receiver is receiver[n].  Increment count[n].
1565     int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row));
1566     increment_mdp_data_at(mdp, count_offset, reg_tmp);
1567     b(done);
1568 
1569     bind(next_test);
1570     // reg_tmp now contains the receiver from the CallData.
1571 
1572     if (row == start_row) {
1573       Label found_null;
1574       // Failed the equality check on receiver[n]...  Test for null.
1575       if (start_row == last_row) {
1576         // The only thing left to do is handle the null case.
1577         if (is_virtual_call) {
1578           cbz(reg_tmp, found_null);
1579           // Receiver did not match any saved receiver and there is no empty row for it.
1580           // Increment total counter to indicate polymorphic case.
1581           increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp);
1582           b(done);
1583           bind(found_null);
1584         } else {
1585           cbnz(reg_tmp, done);
1586         }
1587         break;
1588       }
1589       // Since null is rare, make it be the branch-taken case.
1590       cbz(reg_tmp, found_null);
1591 
1592       // Put all the "Case 3" tests here.
1593       record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call);
1594 
1595       // Found a null.  Keep searching for a matching receiver,
1596       // but remember that this is an empty (unused) slot.
1597       bind(found_null);
1598     }
1599   }
1600 
1601   // In the fall-through case, we found no matching receiver, but we
1602   // observed the receiver[start_row] is NULL.
1603 
1604   // Fill in the receiver field and increment the count.
1605   int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row));
1606   set_mdp_data_at(mdp, recvr_offset, receiver);
1607   int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row));
1608   mov(reg_tmp, DataLayout::counter_increment);
1609   set_mdp_data_at(mdp, count_offset, reg_tmp);
1610   if (start_row > 0) {
1611     b(done);
1612   }
1613 }
1614 
1615 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1616                                                         Register mdp,
1617                                                         Register reg_tmp,
1618                                                         bool is_virtual_call) {
1619   assert(ProfileInterpreter, "must be profiling");
1620   assert_different_registers(receiver, mdp, reg_tmp);
1621 
1622   Label done;
1623 
1624   record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call);
1625 
1626   bind (done);
1627 }
1628 
1629 // Sets mdp, blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
1630 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) {
1631   assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3);
1632 
1633   if (ProfileInterpreter) {
1634     Label profile_continue;
1635     uint row;
1636 
1637     // If no method data exists, go to profile_continue.
1638     test_method_data_pointer(mdp, profile_continue);
1639 
1640     // Update the total ret count.
1641     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1642 
1643     for (row = 0; row < RetData::row_limit(); row++) {
1644       Label next_test;
1645 
1646       // See if return_bci is equal to bci[n]:
1647       test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci,
1648                        Rtemp, next_test);
1649 
1650       // return_bci is equal to bci[n].  Increment the count.
1651       increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp);
1652 
1653       // The method data pointer needs to be updated to reflect the new target.
1654       update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp);
1655       b(profile_continue);
1656       bind(next_test);
1657     }
1658 
1659     update_mdp_for_ret(return_bci);
1660 
1661     bind(profile_continue);
1662   }
1663 }
1664 
1665 
1666 // Sets mdp.
1667 void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
1668   if (ProfileInterpreter) {
1669     Label profile_continue;
1670 
1671     // If no method data exists, go to profile_continue.
1672     test_method_data_pointer(mdp, profile_continue);
1673 
1674     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1675 
1676     // The method data pointer needs to be updated.
1677     int mdp_delta = in_bytes(BitData::bit_data_size());
1678     if (TypeProfileCasts) {
1679       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1680     }
1681     update_mdp_by_constant(mdp, mdp_delta);
1682 
1683     bind (profile_continue);
1684   }
1685 }
1686 
1687 
1688 // Sets mdp, blows Rtemp.
1689 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) {
1690   assert_different_registers(mdp, Rtemp);
1691 
1692   if (ProfileInterpreter && TypeProfileCasts) {
1693     Label profile_continue;
1694 
1695     // If no method data exists, go to profile_continue.
1696     test_method_data_pointer(mdp, profile_continue);
1697 
1698     int count_offset = in_bytes(CounterData::count_offset());
1699     // Back up the address, since we have already bumped the mdp.
1700     count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
1701 
1702     // *Decrement* the counter.  We expect to see zero or small negatives.
1703     increment_mdp_data_at(mdp, count_offset, Rtemp, true);
1704 
1705     bind (profile_continue);
1706   }
1707 }
1708 
1709 
1710 // Sets mdp, blows Rtemp.
1711 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass)
1712 {
1713   assert_different_registers(mdp, klass, Rtemp);
1714 
1715   if (ProfileInterpreter) {
1716     Label profile_continue;
1717 
1718     // If no method data exists, go to profile_continue.
1719     test_method_data_pointer(mdp, profile_continue);
1720 
1721     // The method data pointer needs to be updated.
1722     int mdp_delta = in_bytes(BitData::bit_data_size());
1723     if (TypeProfileCasts) {
1724       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1725 
1726       // Record the object type.
1727       record_klass_in_profile(klass, mdp, Rtemp, false);
1728     }
1729     update_mdp_by_constant(mdp, mdp_delta);
1730 
1731     bind(profile_continue);
1732   }
1733 }
1734 
1735 
1736 // Sets mdp, blows Rtemp.
1737 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1738   assert_different_registers(mdp, Rtemp);
1739 
1740   if (ProfileInterpreter) {
1741     Label profile_continue;
1742 
1743     // If no method data exists, go to profile_continue.
1744     test_method_data_pointer(mdp, profile_continue);
1745 
1746     // Update the default case count
1747     increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp);
1748 
1749     // The method data pointer needs to be updated.
1750     update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp);
1751 
1752     bind(profile_continue);
1753   }
1754 }
1755 
1756 
1757 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2.
1758 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) {
1759   assert_different_registers(mdp, reg_tmp1, reg_tmp2);
1760   assert_different_registers(mdp, reg_tmp1, index);
1761 
1762   if (ProfileInterpreter) {
1763     Label profile_continue;
1764 
1765     const int count_offset = in_bytes(MultiBranchData::case_array_offset()) +
1766                               in_bytes(MultiBranchData::relative_count_offset());
1767 
1768     const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) +
1769                               in_bytes(MultiBranchData::relative_displacement_offset());
1770 
1771     // If no method data exists, go to profile_continue.
1772     test_method_data_pointer(mdp, profile_continue);
1773 
1774     // Build the base (index * per_case_size_in_bytes())
1775     logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size())));
1776 
1777     // Update the case count
1778     add(reg_tmp1, reg_tmp1, count_offset);
1779     increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2);
1780 
1781     // The method data pointer needs to be updated.
1782     add(reg_tmp1, reg_tmp1, displacement_offset - count_offset);
1783     update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2);
1784 
1785     bind (profile_continue);
1786   }
1787 }
1788 
1789 
1790 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) {
1791 #ifdef AARCH64
1792   rev_w(r, r);
1793 #else
1794   if (VM_Version::supports_rev()) {
1795     rev(r, r);
1796   } else {
1797     eor(rtmp1, r, AsmOperand(r, ror, 16));
1798     mvn(rtmp2, 0x0000ff00);
1799     andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8));
1800     eor(r, rtmp1, AsmOperand(r, ror, 8));
1801   }
1802 #endif // AARCH64
1803 }
1804 
1805 
1806 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) {
1807   const intx addr = (intx) (address_of_counter + offset);
1808 
1809   assert ((addr & 0x3) == 0, "address of counter should be aligned");
1810   const intx offset_mask = right_n_bits(AARCH64_ONLY(12 + 2) NOT_AARCH64(12));
1811 
1812   const address base = (address) (addr & ~offset_mask);
1813   const int offs = (int) (addr & offset_mask);
1814 
1815   const Register addr_base = tmp1;
1816   const Register val = tmp2;
1817 
1818   mov_slow(addr_base, base);
1819   ldr_s32(val, Address(addr_base, offs));
1820 
1821   if (avoid_overflow) {
1822     adds_32(val, val, 1);
1823 #ifdef AARCH64
1824     Label L;
1825     b(L, mi);
1826     str_32(val, Address(addr_base, offs));
1827     bind(L);
1828 #else
1829     str(val, Address(addr_base, offs), pl);
1830 #endif // AARCH64
1831   } else {
1832     add_32(val, val, 1);
1833     str_32(val, Address(addr_base, offs));
1834   }
1835 }
1836 
1837 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) {
1838   if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); }
1839 }
1840 
1841 // Inline assembly for:
1842 //
1843 // if (thread is in interp_only_mode) {
1844 //   InterpreterRuntime::post_method_entry();
1845 // }
1846 // if (DTraceMethodProbes) {
1847 //   SharedRuntime::dtrace_method_entry(method, receiver);
1848 // }
1849 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
1850 //   SharedRuntime::rc_trace_method_entry(method, receiver);
1851 // }
1852 
1853 void InterpreterMacroAssembler::notify_method_entry() {
1854   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1855   // track stack depth.  If it is possible to enter interp_only_mode we add
1856   // the code to check if the event should be sent.
1857   if (can_post_interpreter_events()) {
1858     Label L;
1859 
1860     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
1861     cbz(Rtemp, L);
1862 
1863     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry));
1864 
1865     bind(L);
1866   }
1867 
1868   // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry
1869   if (DTraceMethodProbes) {
1870     Label Lcontinue;
1871 
1872     ldrb_global(Rtemp, (address)&DTraceMethodProbes);
1873     cbz(Rtemp, Lcontinue);
1874 
1875     mov(R0, Rthread);
1876     mov(R1, Rmethod);
1877     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1);
1878 
1879     bind(Lcontinue);
1880   }
1881   // RedefineClasses() tracing support for obsolete method entry
1882   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1883     mov(R0, Rthread);
1884     mov(R1, Rmethod);
1885     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1886                  R0, R1);
1887   }
1888 }
1889 
1890 
1891 void InterpreterMacroAssembler::notify_method_exit(
1892                  TosState state, NotifyMethodExitMode mode,
1893                  bool native, Register result_lo, Register result_hi, FloatRegister result_fp) {
1894   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1895   // track stack depth.  If it is possible to enter interp_only_mode we add
1896   // the code to check if the event should be sent.
1897   if (mode == NotifyJVMTI && can_post_interpreter_events()) {
1898     Label L;
1899     // Note: frame::interpreter_frame_result has a dependency on how the
1900     // method result is saved across the call to post_method_exit. If this
1901     // is changed then the interpreter_frame_result implementation will
1902     // need to be updated too.
1903 
1904     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
1905     cbz(Rtemp, L);
1906 
1907     if (native) {
1908       // For c++ and template interpreter push both result registers on the
1909       // stack in native, we don't know the state.
1910       // On AArch64 result registers are stored into the frame at known locations.
1911       // See frame::interpreter_frame_result for code that gets the result values from here.
1912       assert(result_lo != noreg, "result registers should be defined");
1913 
1914 #ifdef AARCH64
1915       assert(result_hi == noreg, "result_hi is not used on AArch64");
1916       assert(result_fp != fnoreg, "FP result register must be defined");
1917 
1918       str_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize));
1919       str(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize));
1920 #else
1921       assert(result_hi != noreg, "result registers should be defined");
1922 
1923 #ifdef __ABI_HARD__
1924       assert(result_fp != fnoreg, "FP result register must be defined");
1925       sub(SP, SP, 2 * wordSize);
1926       fstd(result_fp, Address(SP));
1927 #endif // __ABI_HARD__
1928 
1929       push(RegisterSet(result_lo) | RegisterSet(result_hi));
1930 #endif // AARCH64
1931 
1932       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1933 
1934 #ifdef AARCH64
1935       ldr_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize));
1936       ldr(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize));
1937 #else
1938       pop(RegisterSet(result_lo) | RegisterSet(result_hi));
1939 #ifdef __ABI_HARD__
1940       fldd(result_fp, Address(SP));
1941       add(SP, SP, 2 * wordSize);
1942 #endif // __ABI_HARD__
1943 #endif // AARCH64
1944 
1945     } else {
1946       // For the template interpreter, the value on tos is the size of the
1947       // state. (c++ interpreter calls jvmti somewhere else).
1948       push(state);
1949       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1950       pop(state);
1951     }
1952 
1953     bind(L);
1954   }
1955 
1956   // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit
1957   if (DTraceMethodProbes) {
1958     Label Lcontinue;
1959 
1960     ldrb_global(Rtemp, (address)&DTraceMethodProbes);
1961     cbz(Rtemp, Lcontinue);
1962 
1963     push(state);
1964 
1965     mov(R0, Rthread);
1966     mov(R1, Rmethod);
1967 
1968     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1);
1969 
1970     pop(state);
1971 
1972     bind(Lcontinue);
1973   }
1974 }
1975 
1976 
1977 #ifndef PRODUCT
1978 
1979 void InterpreterMacroAssembler::trace_state(const char* msg) {
1980   int push_size = save_caller_save_registers();
1981 
1982   Label Lcontinue;
1983   InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n");
1984   InlinedString Lmsg(msg);
1985   InlinedAddress Lprintf((address)printf);
1986 
1987   ldr_literal(R0, Lmsg0);
1988   ldr_literal(R1, Lmsg);
1989   mov(R2, FP);
1990   add(R3, SP, push_size);  // original SP (without saved registers)
1991   ldr_literal(Rtemp, Lprintf);
1992   call(Rtemp);
1993 
1994   b(Lcontinue);
1995 
1996   bind_literal(Lmsg0);
1997   bind_literal(Lmsg);
1998   bind_literal(Lprintf);
1999 
2000 
2001   bind(Lcontinue);
2002 
2003   restore_caller_save_registers();
2004 }
2005 
2006 #endif
2007 
2008 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
2009 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
2010                                                         int increment, Address mask_addr,
2011                                                         Register scratch, Register scratch2,
2012                                                         AsmCondition cond, Label* where) {
2013   // caution: scratch2 and base address of counter_addr can be the same
2014   assert_different_registers(scratch, scratch2);
2015   ldr_u32(scratch, counter_addr);
2016   add(scratch, scratch, increment);
2017   str_32(scratch, counter_addr);
2018 
2019 #ifdef AARCH64
2020   ldr_u32(scratch2, mask_addr);
2021   ands_w(ZR, scratch, scratch2);
2022 #else
2023   ldr(scratch2, mask_addr);
2024   andrs(scratch, scratch, scratch2);
2025 #endif // AARCH64
2026   b(*where, cond);
2027 }
2028 
2029 void InterpreterMacroAssembler::get_method_counters(Register method,
2030                                                     Register Rcounters,
2031                                                     Label& skip,
2032                                                     bool saveRegs,
2033                                                     Register reg1,
2034                                                     Register reg2,
2035                                                     Register reg3) {
2036   const Address method_counters(method, Method::method_counters_offset());
2037   Label has_counters;
2038 
2039   ldr(Rcounters, method_counters);
2040   cbnz(Rcounters, has_counters);
2041 
2042   if (saveRegs) {
2043     // Save and restore in use caller-saved registers since they will be trashed by call_VM
2044     assert(reg1 != noreg, "must specify reg1");
2045     assert(reg2 != noreg, "must specify reg2");
2046 #ifdef AARCH64
2047     assert(reg3 != noreg, "must specify reg3");
2048     stp(reg1, reg2, Address(Rstack_top, -2*wordSize, pre_indexed));
2049     stp(reg3, ZR, Address(Rstack_top, -2*wordSize, pre_indexed));
2050 #else
2051     assert(reg3 == noreg, "must not specify reg3");
2052     push(RegisterSet(reg1) | RegisterSet(reg2));
2053 #endif
2054   }
2055 
2056   mov(R1, method);
2057   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1);
2058 
2059   if (saveRegs) {
2060 #ifdef AARCH64
2061     ldp(reg3, ZR, Address(Rstack_top, 2*wordSize, post_indexed));
2062     ldp(reg1, reg2, Address(Rstack_top, 2*wordSize, post_indexed));
2063 #else
2064     pop(RegisterSet(reg1) | RegisterSet(reg2));
2065 #endif
2066   }
2067 
2068   ldr(Rcounters, method_counters);
2069   cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory
2070 
2071   bind(has_counters);
2072 }