1 /*
   2  * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shared/barrierSet.inline.hpp"
  27 #include "gc/shared/cardTableModRefBS.inline.hpp"
  28 #include "gc/shared/collectedHeap.hpp"
  29 #include "interp_masm_arm.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "interpreter/interpreterRuntime.hpp"
  32 #include "logging/log.hpp"
  33 #include "oops/arrayOop.hpp"
  34 #include "oops/markOop.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/methodData.hpp"
  37 #include "prims/jvmtiExport.hpp"
  38 #include "prims/jvmtiThreadState.hpp"
  39 #include "runtime/basicLock.hpp"
  40 #include "runtime/biasedLocking.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 
  43 #if INCLUDE_ALL_GCS
  44 #include "gc/g1/g1CollectedHeap.inline.hpp"
  45 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  46 #include "gc/g1/heapRegion.hpp"
  47 #endif // INCLUDE_ALL_GCS
  48 
  49 //--------------------------------------------------------------------
  50 // Implementation of InterpreterMacroAssembler
  51 
  52 
  53 
  54 
  55 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) {
  56 }
  57 
  58 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
  59 #if defined(ASSERT) && !defined(AARCH64)
  60   // Ensure that last_sp is not filled.
  61   { Label L;
  62     ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
  63     cbz(Rtemp, L);
  64     stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL");
  65     bind(L);
  66   }
  67 #endif // ASSERT && !AARCH64
  68 
  69   // Rbcp must be saved/restored since it may change due to GC.
  70   save_bcp();
  71 
  72 #ifdef AARCH64
  73   check_no_cached_stack_top(Rtemp);
  74   save_stack_top();
  75   check_extended_sp(Rtemp);
  76   cut_sp_before_call();
  77 #endif // AARCH64
  78 
  79   // super call
  80   MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions);
  81 
  82 #ifdef AARCH64
  83   // Restore SP to extended SP
  84   restore_sp_after_call(Rtemp);
  85   check_stack_top();
  86   clear_cached_stack_top();
  87 #endif // AARCH64
  88 
  89   // Restore interpreter specific registers.
  90   restore_bcp();
  91   restore_method();
  92 }
  93 
  94 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  95   assert(entry, "Entry must have been generated by now");
  96   b(entry);
  97 }
  98 
  99 void InterpreterMacroAssembler::check_and_handle_popframe() {
 100   if (can_pop_frame()) {
 101     Label L;
 102     const Register popframe_cond = R2_tmp;
 103 
 104     // Initiate popframe handling only if it is not already being processed.  If the flag
 105     // has the popframe_processing bit set, it means that this code is called *during* popframe
 106     // handling - we don't want to reenter.
 107 
 108     ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset()));
 109     tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L);
 110     tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L);
 111 
 112     // Call Interpreter::remove_activation_preserving_args_entry() to get the
 113     // address of the same-named entrypoint in the generated interpreter code.
 114     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
 115 
 116     // Call indirectly to avoid generation ordering problem.
 117     jump(R0);
 118 
 119     bind(L);
 120   }
 121 }
 122 
 123 
 124 // Blows R2, Rtemp. Sets TOS cached value.
 125 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
 126   const Register thread_state = R2_tmp;
 127 
 128   ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
 129 
 130   const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset());
 131   const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset());
 132   const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset());
 133 #ifndef AARCH64
 134   const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset()
 135                              + in_ByteSize(wordSize));
 136 #endif // !AARCH64
 137 
 138   Register zero = zero_register(Rtemp);
 139 
 140   switch (state) {
 141     case atos: ldr(R0_tos, oop_addr);
 142                str(zero, oop_addr);
 143                interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 144                break;
 145 
 146 #ifdef AARCH64
 147     case ltos: ldr(R0_tos, val_addr);              break;
 148 #else
 149     case ltos: ldr(R1_tos_hi, val_addr_hi);        // fall through
 150 #endif // AARCH64
 151     case btos:                                     // fall through
 152     case ztos:                                     // fall through
 153     case ctos:                                     // fall through
 154     case stos:                                     // fall through
 155     case itos: ldr_s32(R0_tos, val_addr);          break;
 156 #ifdef __SOFTFP__
 157     case dtos: ldr(R1_tos_hi, val_addr_hi);        // fall through
 158     case ftos: ldr(R0_tos, val_addr);              break;
 159 #else
 160     case ftos: ldr_float (S0_tos, val_addr);       break;
 161     case dtos: ldr_double(D0_tos, val_addr);       break;
 162 #endif // __SOFTFP__
 163     case vtos: /* nothing to do */                 break;
 164     default  : ShouldNotReachHere();
 165   }
 166   // Clean up tos value in the thread object
 167   str(zero, val_addr);
 168 #ifndef AARCH64
 169   str(zero, val_addr_hi);
 170 #endif // !AARCH64
 171 
 172   mov(Rtemp, (int) ilgl);
 173   str_32(Rtemp, tos_addr);
 174 }
 175 
 176 
 177 // Blows R2, Rtemp.
 178 void InterpreterMacroAssembler::check_and_handle_earlyret() {
 179   if (can_force_early_return()) {
 180     Label L;
 181     const Register thread_state = R2_tmp;
 182 
 183     ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
 184     cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit;
 185 
 186     // Initiate earlyret handling only if it is not already being processed.
 187     // If the flag has the earlyret_processing bit set, it means that this code
 188     // is called *during* earlyret handling - we don't want to reenter.
 189 
 190     ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset()));
 191     cmp(Rtemp, JvmtiThreadState::earlyret_pending);
 192     b(L, ne);
 193 
 194     // Call Interpreter::remove_activation_early_entry() to get the address of the
 195     // same-named entrypoint in the generated interpreter code.
 196 
 197     ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset()));
 198     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0);
 199 
 200     jump(R0);
 201 
 202     bind(L);
 203   }
 204 }
 205 
 206 
 207 // Sets reg. Blows Rtemp.
 208 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) {
 209   assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
 210   assert(reg != Rtemp, "should be different registers");
 211 
 212   ldrb(Rtemp, Address(Rbcp, bcp_offset));
 213   ldrb(reg, Address(Rbcp, bcp_offset+1));
 214   orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte));
 215 }
 216 
 217 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) {
 218   assert_different_registers(index, tmp_reg);
 219   if (index_size == sizeof(u2)) {
 220     // load bytes of index separately to avoid unaligned access
 221     ldrb(index, Address(Rbcp, bcp_offset+1));
 222     ldrb(tmp_reg, Address(Rbcp, bcp_offset));
 223     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 224   } else if (index_size == sizeof(u4)) {
 225     // TODO-AARCH64: consider using unaligned access here
 226     ldrb(index, Address(Rbcp, bcp_offset+3));
 227     ldrb(tmp_reg, Address(Rbcp, bcp_offset+2));
 228     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 229     ldrb(tmp_reg, Address(Rbcp, bcp_offset+1));
 230     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 231     ldrb(tmp_reg, Address(Rbcp, bcp_offset));
 232     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 233     // Check if the secondary index definition is still ~x, otherwise
 234     // we have to change the following assembler code to calculate the
 235     // plain index.
 236     assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");
 237     mvn_32(index, index);  // convert to plain index
 238   } else if (index_size == sizeof(u1)) {
 239     ldrb(index, Address(Rbcp, bcp_offset));
 240   } else {
 241     ShouldNotReachHere();
 242   }
 243 }
 244 
 245 // Sets cache, index.
 246 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) {
 247   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 248   assert_different_registers(cache, index);
 249 
 250   get_index_at_bcp(index, bcp_offset, cache, index_size);
 251 
 252   // load constant pool cache pointer
 253   ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
 254 
 255   // convert from field index to ConstantPoolCacheEntry index
 256   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
 257   // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called
 258   logical_shift_left(index, index, 2);
 259 }
 260 
 261 // Sets cache, index, bytecode.
 262 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) {
 263   get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);
 264   // caution index and bytecode can be the same
 265   add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord));
 266 #ifdef AARCH64
 267   add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()));
 268   ldarb(bytecode, bytecode);
 269 #else
 270   ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())));
 271   TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true);
 272 #endif // AARCH64
 273 }
 274 
 275 // Sets cache. Blows reg_tmp.
 276 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) {
 277   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 278   assert_different_registers(cache, reg_tmp);
 279 
 280   get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size);
 281 
 282   // load constant pool cache pointer
 283   ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
 284 
 285   // skip past the header
 286   add(cache, cache, in_bytes(ConstantPoolCache::base_offset()));
 287   // convert from field index to ConstantPoolCacheEntry index
 288   // and from word offset to byte offset
 289   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
 290   add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord));
 291 }
 292 
 293 // Load object from cpool->resolved_references(index)
 294 void InterpreterMacroAssembler::load_resolved_reference_at_index(
 295                                            Register result, Register index) {
 296   assert_different_registers(result, index);
 297   get_constant_pool(result);
 298 
 299   Register cache = result;
 300   // load pointer for resolved_references[] objArray
 301   ldr(cache, Address(result, ConstantPool::resolved_references_offset_in_bytes()));
 302   // JNIHandles::resolve(result)
 303   ldr(cache, Address(cache, 0));
 304   // Add in the index
 305   // convert from field index to resolved_references() index and from
 306   // word index to byte offset. Since this is a java object, it can be compressed
 307   add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop));
 308   load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 309 }
 310 
 311 // Generate a subtype check: branch to not_subtype if sub_klass is
 312 // not a subtype of super_klass.
 313 // Profiling code for the subtype check failure (profile_typecheck_failed)
 314 // should be explicitly generated by the caller in the not_subtype case.
 315 // Blows Rtemp, tmp1, tmp2.
 316 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 317                                                   Register Rsuper_klass,
 318                                                   Label &not_subtype,
 319                                                   Register tmp1,
 320                                                   Register tmp2) {
 321 
 322   assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp);
 323   Label ok_is_subtype, loop, update_cache;
 324 
 325   const Register super_check_offset = tmp1;
 326   const Register cached_super = tmp2;
 327 
 328   // Profile the not-null value's klass.
 329   profile_typecheck(tmp1, Rsub_klass);
 330 
 331   // Load the super-klass's check offset into
 332   ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset()));
 333 
 334   // Check for self
 335   cmp(Rsub_klass, Rsuper_klass);
 336 
 337   // Load from the sub-klass's super-class display list, or a 1-word cache of
 338   // the secondary superclass list, or a failing value with a sentinel offset
 339   // if the super-klass is an interface or exceptionally deep in the Java
 340   // hierarchy and we have to scan the secondary superclass list the hard way.
 341   // See if we get an immediate positive hit
 342   ldr(cached_super, Address(Rsub_klass, super_check_offset));
 343 
 344   cond_cmp(Rsuper_klass, cached_super, ne);
 345   b(ok_is_subtype, eq);
 346 
 347   // Check for immediate negative hit
 348   cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset()));
 349   b(not_subtype, ne);
 350 
 351   // Now do a linear scan of the secondary super-klass chain.
 352   const Register supers_arr = tmp1;
 353   const Register supers_cnt = tmp2;
 354   const Register cur_super  = Rtemp;
 355 
 356   // Load objArrayOop of secondary supers.
 357   ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset()));
 358 
 359   ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length
 360 #ifdef AARCH64
 361   cbz(supers_cnt, not_subtype);
 362   add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes());
 363 #else
 364   cmp(supers_cnt, 0);
 365 
 366   // Skip to the start of array elements and prefetch the first super-klass.
 367   ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne);
 368   b(not_subtype, eq);
 369 #endif // AARCH64
 370 
 371   bind(loop);
 372 
 373 #ifdef AARCH64
 374   ldr(cur_super, Address(supers_arr, wordSize, post_indexed));
 375 #endif // AARCH64
 376 
 377   cmp(cur_super, Rsuper_klass);
 378   b(update_cache, eq);
 379 
 380   subs(supers_cnt, supers_cnt, 1);
 381 
 382 #ifndef AARCH64
 383   ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne);
 384 #endif // !AARCH64
 385 
 386   b(loop, ne);
 387 
 388   b(not_subtype);
 389 
 390   bind(update_cache);
 391   // Must be equal but missed in cache.  Update cache.
 392   str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset()));
 393 
 394   bind(ok_is_subtype);
 395 }
 396 
 397 
 398 // The 1st part of the store check.
 399 // Sets card_table_base register.
 400 void InterpreterMacroAssembler::store_check_part1(Register card_table_base) {
 401   // Check barrier set type (should be card table) and element size
 402   BarrierSet* bs = Universe::heap()->barrier_set();
 403   assert(bs->kind() == BarrierSet::CardTableForRS ||
 404          bs->kind() == BarrierSet::CardTableExtension,
 405          "Wrong barrier set kind");
 406 
 407   CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
 408   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "Adjust store check code");
 409 
 410   // Load card table base address.
 411 
 412   /* Performance note.
 413 
 414      There is an alternative way of loading card table base address
 415      from thread descriptor, which may look more efficient:
 416 
 417      ldr(card_table_base, Address(Rthread, JavaThread::card_table_base_offset()));
 418 
 419      However, performance measurements of micro benchmarks and specJVM98
 420      showed that loading of card table base from thread descriptor is
 421      7-18% slower compared to loading of literal embedded into the code.
 422      Possible cause is a cache miss (card table base address resides in a
 423      rarely accessed area of thread descriptor).
 424   */
 425   // TODO-AARCH64 Investigate if mov_slow is faster than ldr from Rthread on AArch64
 426   mov_address(card_table_base, (address)ct->byte_map_base, symbolic_Relocation::card_table_reference);
 427 }
 428 
 429 // The 2nd part of the store check.
 430 void InterpreterMacroAssembler::store_check_part2(Register obj, Register card_table_base, Register tmp) {
 431   assert_different_registers(obj, card_table_base, tmp);
 432 
 433   assert(CardTableModRefBS::dirty_card_val() == 0, "Dirty card value must be 0 due to optimizations.");
 434 #ifdef AARCH64
 435   add(card_table_base, card_table_base, AsmOperand(obj, lsr, CardTableModRefBS::card_shift));
 436   Address card_table_addr(card_table_base);
 437 #else
 438   Address card_table_addr(card_table_base, obj, lsr, CardTableModRefBS::card_shift);
 439 #endif
 440 
 441   if (UseCondCardMark) {
 442     if (UseConcMarkSweepGC) {
 443       membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), noreg);
 444     }
 445     Label already_dirty;
 446 
 447     ldrb(tmp, card_table_addr);
 448     cbz(tmp, already_dirty);
 449 
 450     set_card(card_table_base, card_table_addr, tmp);
 451     bind(already_dirty);
 452 
 453   } else {
 454     if (UseConcMarkSweepGC && CMSPrecleaningEnabled) {
 455       membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg);
 456     }
 457     set_card(card_table_base, card_table_addr, tmp);
 458   }
 459 }
 460 
 461 void InterpreterMacroAssembler::set_card(Register card_table_base, Address card_table_addr, Register tmp) {
 462 #ifdef AARCH64
 463   strb(ZR, card_table_addr);
 464 #else
 465   CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set());
 466   if ((((uintptr_t)ct->byte_map_base & 0xff) == 0)) {
 467     // Card table is aligned so the lowest byte of the table address base is zero.
 468     // This works only if the code is not saved for later use, possibly
 469     // in a context where the base would no longer be aligned.
 470     strb(card_table_base, card_table_addr);
 471   } else {
 472     mov(tmp, 0);
 473     strb(tmp, card_table_addr);
 474   }
 475 #endif // AARCH64
 476 }
 477 
 478 //////////////////////////////////////////////////////////////////////////////////
 479 #if INCLUDE_ALL_GCS
 480 
 481 // G1 pre-barrier.
 482 // Blows all volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
 483 // If store_addr != noreg, then previous value is loaded from [store_addr];
 484 // in such case store_addr and new_val registers are preserved;
 485 // otherwise pre_val register is preserved.
 486 void InterpreterMacroAssembler::g1_write_barrier_pre(Register store_addr,
 487                                                      Register new_val,
 488                                                      Register pre_val,
 489                                                      Register tmp1,
 490                                                      Register tmp2) {
 491   Label done;
 492   Label runtime;
 493 
 494   if (store_addr != noreg) {
 495     assert_different_registers(store_addr, new_val, pre_val, tmp1, tmp2, noreg);
 496   } else {
 497     assert (new_val == noreg, "should be");
 498     assert_different_registers(pre_val, tmp1, tmp2, noreg);
 499   }
 500 
 501   Address in_progress(Rthread, in_bytes(JavaThread::satb_mark_queue_offset() +
 502                                         SATBMarkQueue::byte_offset_of_active()));
 503   Address index(Rthread, in_bytes(JavaThread::satb_mark_queue_offset() +
 504                                   SATBMarkQueue::byte_offset_of_index()));
 505   Address buffer(Rthread, in_bytes(JavaThread::satb_mark_queue_offset() +
 506                                    SATBMarkQueue::byte_offset_of_buf()));
 507 
 508   // Is marking active?
 509   assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "adjust this code");
 510   ldrb(tmp1, in_progress);
 511   cbz(tmp1, done);
 512 
 513   // Do we need to load the previous value?
 514   if (store_addr != noreg) {
 515     load_heap_oop(pre_val, Address(store_addr, 0));
 516   }
 517 
 518   // Is the previous value null?
 519   cbz(pre_val, done);
 520 
 521   // Can we store original value in the thread's buffer?
 522   // Is index == 0?
 523   // (The index field is typed as size_t.)
 524 
 525   ldr(tmp1, index);           // tmp1 := *index_adr
 526   ldr(tmp2, buffer);
 527 
 528   subs(tmp1, tmp1, wordSize); // tmp1 := tmp1 - wordSize
 529   b(runtime, lt);             // If negative, goto runtime
 530 
 531   str(tmp1, index);           // *index_adr := tmp1
 532 
 533   // Record the previous value
 534   str(pre_val, Address(tmp2, tmp1));
 535   b(done);
 536 
 537   bind(runtime);
 538 
 539   // save the live input values
 540 #ifdef AARCH64
 541   if (store_addr != noreg) {
 542     raw_push(store_addr, new_val);
 543   } else {
 544     raw_push(pre_val, ZR);
 545   }
 546 #else
 547   if (store_addr != noreg) {
 548     // avoid raw_push to support any ordering of store_addr and new_val
 549     push(RegisterSet(store_addr) | RegisterSet(new_val));
 550   } else {
 551     push(pre_val);
 552   }
 553 #endif // AARCH64
 554 
 555   if (pre_val != R0) {
 556     mov(R0, pre_val);
 557   }
 558   mov(R1, Rthread);
 559 
 560   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), R0, R1);
 561 
 562 #ifdef AARCH64
 563   if (store_addr != noreg) {
 564     raw_pop(store_addr, new_val);
 565   } else {
 566     raw_pop(pre_val, ZR);
 567   }
 568 #else
 569   if (store_addr != noreg) {
 570     pop(RegisterSet(store_addr) | RegisterSet(new_val));
 571   } else {
 572     pop(pre_val);
 573   }
 574 #endif // AARCH64
 575 
 576   bind(done);
 577 }
 578 
 579 // G1 post-barrier.
 580 // Blows all volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
 581 void InterpreterMacroAssembler::g1_write_barrier_post(Register store_addr,
 582                                                       Register new_val,
 583                                                       Register tmp1,
 584                                                       Register tmp2,
 585                                                       Register tmp3) {
 586 
 587   Address queue_index(Rthread, in_bytes(JavaThread::dirty_card_queue_offset() +
 588                                         DirtyCardQueue::byte_offset_of_index()));
 589   Address buffer(Rthread, in_bytes(JavaThread::dirty_card_queue_offset() +
 590                                    DirtyCardQueue::byte_offset_of_buf()));
 591 
 592   BarrierSet* bs = Universe::heap()->barrier_set();
 593   CardTableModRefBS* ct = (CardTableModRefBS*)bs;
 594   Label done;
 595   Label runtime;
 596 
 597   // Does store cross heap regions?
 598 
 599   eor(tmp1, store_addr, new_val);
 600 #ifdef AARCH64
 601   logical_shift_right(tmp1, tmp1, HeapRegion::LogOfHRGrainBytes);
 602   cbz(tmp1, done);
 603 #else
 604   movs(tmp1, AsmOperand(tmp1, lsr, HeapRegion::LogOfHRGrainBytes));
 605   b(done, eq);
 606 #endif
 607 
 608   // crosses regions, storing NULL?
 609 
 610   cbz(new_val, done);
 611 
 612   // storing region crossing non-NULL, is card already dirty?
 613   const Register card_addr = tmp1;
 614   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
 615 
 616   mov_address(tmp2, (address)ct->byte_map_base, symbolic_Relocation::card_table_reference);
 617   add(card_addr, tmp2, AsmOperand(store_addr, lsr, CardTableModRefBS::card_shift));
 618 
 619   ldrb(tmp2, Address(card_addr));
 620   cmp(tmp2, (int)G1SATBCardTableModRefBS::g1_young_card_val());
 621   b(done, eq);
 622 
 623   membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp2);
 624 
 625   assert(CardTableModRefBS::dirty_card_val() == 0, "adjust this code");
 626   ldrb(tmp2, Address(card_addr));
 627   cbz(tmp2, done);
 628 
 629   // storing a region crossing, non-NULL oop, card is clean.
 630   // dirty card and log.
 631 
 632   strb(zero_register(tmp2), Address(card_addr));
 633 
 634   ldr(tmp2, queue_index);
 635   ldr(tmp3, buffer);
 636 
 637   subs(tmp2, tmp2, wordSize);
 638   b(runtime, lt); // go to runtime if now negative
 639 
 640   str(tmp2, queue_index);
 641 
 642   str(card_addr, Address(tmp3, tmp2));
 643   b(done);
 644 
 645   bind(runtime);
 646 
 647   if (card_addr != R0) {
 648     mov(R0, card_addr);
 649   }
 650   mov(R1, Rthread);
 651   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), R0, R1);
 652 
 653   bind(done);
 654 }
 655 
 656 #endif // INCLUDE_ALL_GCS
 657 //////////////////////////////////////////////////////////////////////////////////
 658 
 659 
 660 // Java Expression Stack
 661 
 662 void InterpreterMacroAssembler::pop_ptr(Register r) {
 663   assert(r != Rstack_top, "unpredictable instruction");
 664   ldr(r, Address(Rstack_top, wordSize, post_indexed));
 665 }
 666 
 667 void InterpreterMacroAssembler::pop_i(Register r) {
 668   assert(r != Rstack_top, "unpredictable instruction");
 669   ldr_s32(r, Address(Rstack_top, wordSize, post_indexed));
 670   zap_high_non_significant_bits(r);
 671 }
 672 
 673 #ifdef AARCH64
 674 void InterpreterMacroAssembler::pop_l(Register r) {
 675   assert(r != Rstack_top, "unpredictable instruction");
 676   ldr(r, Address(Rstack_top, 2*wordSize, post_indexed));
 677 }
 678 #else
 679 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) {
 680   assert_different_registers(lo, hi);
 681   assert(lo < hi, "lo must be < hi");
 682   pop(RegisterSet(lo) | RegisterSet(hi));
 683 }
 684 #endif // AARCH64
 685 
 686 void InterpreterMacroAssembler::pop_f(FloatRegister fd) {
 687 #ifdef AARCH64
 688   ldr_s(fd, Address(Rstack_top, wordSize, post_indexed));
 689 #else
 690   fpops(fd);
 691 #endif // AARCH64
 692 }
 693 
 694 void InterpreterMacroAssembler::pop_d(FloatRegister fd) {
 695 #ifdef AARCH64
 696   ldr_d(fd, Address(Rstack_top, 2*wordSize, post_indexed));
 697 #else
 698   fpopd(fd);
 699 #endif // AARCH64
 700 }
 701 
 702 
 703 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value.
 704 void InterpreterMacroAssembler::pop(TosState state) {
 705   switch (state) {
 706     case atos: pop_ptr(R0_tos);                              break;
 707     case btos:                                               // fall through
 708     case ztos:                                               // fall through
 709     case ctos:                                               // fall through
 710     case stos:                                               // fall through
 711     case itos: pop_i(R0_tos);                                break;
 712 #ifdef AARCH64
 713     case ltos: pop_l(R0_tos);                                break;
 714 #else
 715     case ltos: pop_l(R0_tos_lo, R1_tos_hi);                  break;
 716 #endif // AARCH64
 717 #ifdef __SOFTFP__
 718     case ftos: pop_i(R0_tos);                                break;
 719     case dtos: pop_l(R0_tos_lo, R1_tos_hi);                  break;
 720 #else
 721     case ftos: pop_f(S0_tos);                                break;
 722     case dtos: pop_d(D0_tos);                                break;
 723 #endif // __SOFTFP__
 724     case vtos: /* nothing to do */                           break;
 725     default  : ShouldNotReachHere();
 726   }
 727   interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 728 }
 729 
 730 void InterpreterMacroAssembler::push_ptr(Register r) {
 731   assert(r != Rstack_top, "unpredictable instruction");
 732   str(r, Address(Rstack_top, -wordSize, pre_indexed));
 733   check_stack_top_on_expansion();
 734 }
 735 
 736 void InterpreterMacroAssembler::push_i(Register r) {
 737   assert(r != Rstack_top, "unpredictable instruction");
 738   str_32(r, Address(Rstack_top, -wordSize, pre_indexed));
 739   check_stack_top_on_expansion();
 740 }
 741 
 742 #ifdef AARCH64
 743 void InterpreterMacroAssembler::push_l(Register r) {
 744   assert(r != Rstack_top, "unpredictable instruction");
 745   str(r, Address(Rstack_top, -2*wordSize, pre_indexed));
 746   check_stack_top_on_expansion();
 747 }
 748 #else
 749 void InterpreterMacroAssembler::push_l(Register lo, Register hi) {
 750   assert_different_registers(lo, hi);
 751   assert(lo < hi, "lo must be < hi");
 752   push(RegisterSet(lo) | RegisterSet(hi));
 753 }
 754 #endif // AARCH64
 755 
 756 void InterpreterMacroAssembler::push_f() {
 757 #ifdef AARCH64
 758   str_s(S0_tos, Address(Rstack_top, -wordSize, pre_indexed));
 759   check_stack_top_on_expansion();
 760 #else
 761   fpushs(S0_tos);
 762 #endif // AARCH64
 763 }
 764 
 765 void InterpreterMacroAssembler::push_d() {
 766 #ifdef AARCH64
 767   str_d(D0_tos, Address(Rstack_top, -2*wordSize, pre_indexed));
 768   check_stack_top_on_expansion();
 769 #else
 770   fpushd(D0_tos);
 771 #endif // AARCH64
 772 }
 773 
 774 // Transition state -> vtos. Blows Rtemp.
 775 void InterpreterMacroAssembler::push(TosState state) {
 776   interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 777   switch (state) {
 778     case atos: push_ptr(R0_tos);                              break;
 779     case btos:                                                // fall through
 780     case ztos:                                                // fall through
 781     case ctos:                                                // fall through
 782     case stos:                                                // fall through
 783     case itos: push_i(R0_tos);                                break;
 784 #ifdef AARCH64
 785     case ltos: push_l(R0_tos);                                break;
 786 #else
 787     case ltos: push_l(R0_tos_lo, R1_tos_hi);                  break;
 788 #endif // AARCH64
 789 #ifdef __SOFTFP__
 790     case ftos: push_i(R0_tos);                                break;
 791     case dtos: push_l(R0_tos_lo, R1_tos_hi);                  break;
 792 #else
 793     case ftos: push_f();                                      break;
 794     case dtos: push_d();                                      break;
 795 #endif // __SOFTFP__
 796     case vtos: /* nothing to do */                            break;
 797     default  : ShouldNotReachHere();
 798   }
 799 }
 800 
 801 
 802 #ifndef AARCH64
 803 
 804 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value.
 805 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) {
 806 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 807   // According to interpreter calling conventions, result is returned in R0/R1,
 808   // but templates expect ftos in S0, and dtos in D0.
 809   if (state == ftos) {
 810     fmsr(S0_tos, R0);
 811   } else if (state == dtos) {
 812     fmdrr(D0_tos, R0, R1);
 813   }
 814 #endif // !__SOFTFP__ && !__ABI_HARD__
 815 }
 816 
 817 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions).
 818 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) {
 819 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 820   // According to interpreter calling conventions, result is returned in R0/R1,
 821   // so ftos (S0) and dtos (D0) are moved to R0/R1.
 822   if (state == ftos) {
 823     fmrs(R0, S0_tos);
 824   } else if (state == dtos) {
 825     fmrrd(R0, R1, D0_tos);
 826   }
 827 #endif // !__SOFTFP__ && !__ABI_HARD__
 828 }
 829 
 830 #endif // !AARCH64
 831 
 832 
 833 // Helpers for swap and dup
 834 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
 835   ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
 836 }
 837 
 838 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
 839   str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
 840 }
 841 
 842 
 843 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
 844 #ifdef AARCH64
 845   check_no_cached_stack_top(Rtemp);
 846   save_stack_top();
 847   cut_sp_before_call();
 848   mov(Rparams, Rstack_top);
 849 #endif // AARCH64
 850 
 851   // set sender sp
 852   mov(Rsender_sp, SP);
 853 
 854 #ifndef AARCH64
 855   // record last_sp
 856   str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
 857 #endif // !AARCH64
 858 }
 859 
 860 // Jump to from_interpreted entry of a call unless single stepping is possible
 861 // in this thread in which case we must call the i2i entry
 862 void InterpreterMacroAssembler::jump_from_interpreted(Register method) {
 863   assert_different_registers(method, Rtemp);
 864 
 865   prepare_to_jump_from_interpreted();
 866 
 867   if (can_post_interpreter_events()) {
 868     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
 869     // compiled code in threads for which the event is enabled.  Check here for
 870     // interp_only_mode if these events CAN be enabled.
 871 
 872     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
 873 #ifdef AARCH64
 874     {
 875       Label not_interp_only_mode;
 876 
 877       cbz(Rtemp, not_interp_only_mode);
 878       indirect_jump(Address(method, Method::interpreter_entry_offset()), Rtemp);
 879 
 880       bind(not_interp_only_mode);
 881     }
 882 #else
 883     cmp(Rtemp, 0);
 884     ldr(PC, Address(method, Method::interpreter_entry_offset()), ne);
 885 #endif // AARCH64
 886   }
 887 
 888   indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp);
 889 }
 890 
 891 
 892 void InterpreterMacroAssembler::restore_dispatch() {
 893   mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos));
 894 }
 895 
 896 
 897 // The following two routines provide a hook so that an implementation
 898 // can schedule the dispatch in two parts.
 899 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
 900   // Nothing ARM-specific to be done here.
 901 }
 902 
 903 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
 904   dispatch_next(state, step);
 905 }
 906 
 907 void InterpreterMacroAssembler::dispatch_base(TosState state,
 908                                               DispatchTableMode table_mode,
 909                                               bool verifyoop) {
 910   if (VerifyActivationFrameSize) {
 911     Label L;
 912 #ifdef AARCH64
 913     mov(Rtemp, SP);
 914     sub(Rtemp, FP, Rtemp);
 915 #else
 916     sub(Rtemp, FP, SP);
 917 #endif // AARCH64
 918     int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
 919     cmp(Rtemp, min_frame_size);
 920     b(L, ge);
 921     stop("broken stack frame");
 922     bind(L);
 923   }
 924 
 925   if (verifyoop) {
 926     interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 927   }
 928 
 929   if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) {
 930     zap_high_non_significant_bits(R0_tos);
 931   }
 932 
 933 #ifdef ASSERT
 934   Label L;
 935   mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos));
 936   cmp(Rtemp, RdispatchTable);
 937   b(L, eq);
 938   stop("invalid RdispatchTable");
 939   bind(L);
 940 #endif
 941 
 942   if (table_mode == DispatchDefault) {
 943     if (state == vtos) {
 944       indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp);
 945     } else {
 946 #ifdef AARCH64
 947       sub(Rtemp, R3_bytecode, (Interpreter::distance_from_dispatch_table(vtos) -
 948                            Interpreter::distance_from_dispatch_table(state)));
 949       indirect_jump(Address::indexed_ptr(RdispatchTable, Rtemp), Rtemp);
 950 #else
 951       // on 32-bit ARM this method is faster than the one above.
 952       sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) -
 953                            Interpreter::distance_from_dispatch_table(state)) * wordSize);
 954       indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 955 #endif
 956     }
 957   } else {
 958     assert(table_mode == DispatchNormal, "invalid dispatch table mode");
 959     address table = (address) Interpreter::normal_table(state);
 960     mov_slow(Rtemp, table);
 961     indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 962   }
 963 
 964   nop(); // to avoid filling CPU pipeline with invalid instructions
 965   nop();
 966 }
 967 
 968 void InterpreterMacroAssembler::dispatch_only(TosState state) {
 969   dispatch_base(state, DispatchDefault);
 970 }
 971 
 972 
 973 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
 974   dispatch_base(state, DispatchNormal);
 975 }
 976 
 977 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
 978   dispatch_base(state, DispatchNormal, false);
 979 }
 980 
 981 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) {
 982   // load next bytecode and advance Rbcp
 983   ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed));
 984   dispatch_base(state, DispatchDefault);
 985 }
 986 
 987 void InterpreterMacroAssembler::narrow(Register result) {
 988   // mask integer result to narrower return type.
 989   const Register Rtmp = R2;
 990 
 991   // get method type
 992   ldr(Rtmp, Address(Rmethod, Method::const_offset()));
 993   ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset()));
 994 
 995   Label notBool, notByte, notChar, done;
 996   cmp(Rtmp, T_INT);
 997   b(done, eq);
 998 
 999   cmp(Rtmp, T_BOOLEAN);
1000   b(notBool, ne);
1001   and_32(result, result, 1);
1002   b(done);
1003 
1004   bind(notBool);
1005   cmp(Rtmp, T_BYTE);
1006   b(notByte, ne);
1007   sign_extend(result, result, 8);
1008   b(done);
1009 
1010   bind(notByte);
1011   cmp(Rtmp, T_CHAR);
1012   b(notChar, ne);
1013   zero_extend(result, result, 16);
1014   b(done);
1015 
1016   bind(notChar);
1017   // cmp(Rtmp, T_SHORT);
1018   // b(done, ne);
1019   sign_extend(result, result, 16);
1020 
1021   // Nothing to do
1022   bind(done);
1023 }
1024 
1025 // remove activation
1026 //
1027 // Unlock the receiver if this is a synchronized method.
1028 // Unlock any Java monitors from syncronized blocks.
1029 // Remove the activation from the stack.
1030 //
1031 // If there are locked Java monitors
1032 //    If throw_monitor_exception
1033 //       throws IllegalMonitorStateException
1034 //    Else if install_monitor_exception
1035 //       installs IllegalMonitorStateException
1036 //    Else
1037 //       no error processing
1038 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr,
1039                                                   bool throw_monitor_exception,
1040                                                   bool install_monitor_exception,
1041                                                   bool notify_jvmdi) {
1042   Label unlock, unlocked, no_unlock;
1043 
1044   // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result.
1045 
1046   const Address do_not_unlock_if_synchronized(Rthread,
1047                          JavaThread::do_not_unlock_if_synchronized_offset());
1048 
1049   const Register Rflag = R2;
1050   const Register Raccess_flags = R3;
1051 
1052   restore_method();
1053 
1054   ldrb(Rflag, do_not_unlock_if_synchronized);
1055 
1056   // get method access flags
1057   ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset()));
1058 
1059   strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag
1060 
1061   // check if method is synchronized
1062 
1063   tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked);
1064 
1065   // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set.
1066   cbnz(Rflag, no_unlock);
1067 
1068   // unlock monitor
1069   push(state);                                   // save result
1070 
1071   // BasicObjectLock will be first in list, since this is a synchronized method. However, need
1072   // to check that the object has not been unlocked by an explicit monitorexit bytecode.
1073 
1074   const Register Rmonitor = R1;                  // fixed in unlock_object()
1075   const Register Robj = R2;
1076 
1077   // address of first monitor
1078   sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock));
1079 
1080   ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes()));
1081   cbnz(Robj, unlock);
1082 
1083   pop(state);
1084 
1085   if (throw_monitor_exception) {
1086     // Entry already unlocked, need to throw exception
1087     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
1088     should_not_reach_here();
1089   } else {
1090     // Monitor already unlocked during a stack unroll.
1091     // If requested, install an illegal_monitor_state_exception.
1092     // Continue with stack unrolling.
1093     if (install_monitor_exception) {
1094       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
1095     }
1096     b(unlocked);
1097   }
1098 
1099 
1100   // Exception case for the check that all monitors are unlocked.
1101   const Register Rcur = R2;
1102   Label restart_check_monitors_unlocked, exception_monitor_is_still_locked;
1103 
1104   bind(exception_monitor_is_still_locked);
1105   // Monitor entry is still locked, need to throw exception.
1106   // Rcur: monitor entry.
1107 
1108   if (throw_monitor_exception) {
1109     // Throw exception
1110     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
1111     should_not_reach_here();
1112   } else {
1113     // Stack unrolling. Unlock object and install illegal_monitor_exception
1114     // Unlock does not block, so don't have to worry about the frame
1115 
1116     push(state);
1117     mov(R1, Rcur);
1118     unlock_object(R1);
1119 
1120     if (install_monitor_exception) {
1121       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
1122     }
1123 
1124     pop(state);
1125     b(restart_check_monitors_unlocked);
1126   }
1127 
1128   bind(unlock);
1129   unlock_object(Rmonitor);
1130   pop(state);
1131 
1132   // Check that for block-structured locking (i.e., that all locked objects has been unlocked)
1133   bind(unlocked);
1134 
1135   // Check that all monitors are unlocked
1136   {
1137     Label loop;
1138 
1139     const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
1140     const Register Rbottom = R3;
1141     const Register Rcur_obj = Rtemp;
1142 
1143     bind(restart_check_monitors_unlocked);
1144 
1145     ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
1146                                  // points to current entry, starting with top-most entry
1147     sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize);
1148                                  // points to word before bottom of monitor block
1149 
1150     cmp(Rcur, Rbottom);          // check if there are no monitors
1151 #ifndef AARCH64
1152     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
1153                                  // prefetch monitor's object
1154 #endif // !AARCH64
1155     b(no_unlock, eq);
1156 
1157     bind(loop);
1158 #ifdef AARCH64
1159     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()));
1160 #endif // AARCH64
1161     // check if current entry is used
1162     cbnz(Rcur_obj, exception_monitor_is_still_locked);
1163 
1164     add(Rcur, Rcur, entry_size);      // otherwise advance to next entry
1165     cmp(Rcur, Rbottom);               // check if bottom reached
1166 #ifndef AARCH64
1167     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
1168                                       // prefetch monitor's object
1169 #endif // !AARCH64
1170     b(loop, ne);                      // if not at bottom then check this entry
1171   }
1172 
1173   bind(no_unlock);
1174 
1175   // jvmti support
1176   if (notify_jvmdi) {
1177     notify_method_exit(state, NotifyJVMTI);     // preserve TOSCA
1178   } else {
1179     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
1180   }
1181 
1182   // remove activation
1183 #ifdef AARCH64
1184   ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
1185   ldp(FP, LR, Address(FP));
1186   mov(SP, Rtemp);
1187 #else
1188   mov(Rtemp, FP);
1189   ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
1190   ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
1191 #endif
1192 
1193   if (ret_addr != LR) {
1194     mov(ret_addr, LR);
1195   }
1196 }
1197 
1198 
1199 // At certain points in the method invocation the monitor of
1200 // synchronized methods hasn't been entered yet.
1201 // To correctly handle exceptions at these points, we set the thread local
1202 // variable _do_not_unlock_if_synchronized to true. The remove_activation will
1203 // check this flag.
1204 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) {
1205   const Address do_not_unlock_if_synchronized(Rthread,
1206                          JavaThread::do_not_unlock_if_synchronized_offset());
1207   if (flag) {
1208     mov(tmp, 1);
1209     strb(tmp, do_not_unlock_if_synchronized);
1210   } else {
1211     strb(zero_register(tmp), do_not_unlock_if_synchronized);
1212   }
1213 }
1214 
1215 // Lock object
1216 //
1217 // Argument: R1 : Points to BasicObjectLock to be used for locking.
1218 // Must be initialized with object to lock.
1219 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM.
1220 void InterpreterMacroAssembler::lock_object(Register Rlock) {
1221   assert(Rlock == R1, "the second argument");
1222 
1223   if (UseHeavyMonitors) {
1224     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
1225   } else {
1226     Label done;
1227 
1228     const Register Robj = R2;
1229     const Register Rmark = R3;
1230     assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp);
1231 
1232     const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1233     const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1234     const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
1235 
1236     Label already_locked, slow_case;
1237 
1238     // Load object pointer
1239     ldr(Robj, Address(Rlock, obj_offset));
1240 
1241     if (UseBiasedLocking) {
1242       biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case);
1243     }
1244 
1245 #ifdef AARCH64
1246     assert(oopDesc::mark_offset_in_bytes() == 0, "must be");
1247     ldr(Rmark, Robj);
1248 
1249     // Test if object is already locked
1250     assert(markOopDesc::unlocked_value == 1, "adjust this code");
1251     tbz(Rmark, exact_log2(markOopDesc::unlocked_value), already_locked);
1252 
1253 #else // AARCH64
1254 
1255     // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
1256     // That would be acceptable as ether CAS or slow case path is taken in that case.
1257     // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as
1258     // loads are satisfied from a store queue if performed on the same processor).
1259 
1260     assert(oopDesc::mark_offset_in_bytes() == 0, "must be");
1261     ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes()));
1262 
1263     // Test if object is already locked
1264     tst(Rmark, markOopDesc::unlocked_value);
1265     b(already_locked, eq);
1266 
1267 #endif // !AARCH64
1268     // Save old object->mark() into BasicLock's displaced header
1269     str(Rmark, Address(Rlock, mark_offset));
1270 
1271     cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case);
1272 
1273 #ifndef PRODUCT
1274     if (PrintBiasedLockingStatistics) {
1275       cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr());
1276     }
1277 #endif //!PRODUCT
1278 
1279     b(done);
1280 
1281     // If we got here that means the object is locked by ether calling thread or another thread.
1282     bind(already_locked);
1283     // Handling of locked objects: recursive locks and slow case.
1284 
1285     // Fast check for recursive lock.
1286     //
1287     // Can apply the optimization only if this is a stack lock
1288     // allocated in this thread. For efficiency, we can focus on
1289     // recently allocated stack locks (instead of reading the stack
1290     // base and checking whether 'mark' points inside the current
1291     // thread stack):
1292     //  1) (mark & 3) == 0
1293     //  2) SP <= mark < SP + os::pagesize()
1294     //
1295     // Warning: SP + os::pagesize can overflow the stack base. We must
1296     // neither apply the optimization for an inflated lock allocated
1297     // just above the thread stack (this is why condition 1 matters)
1298     // nor apply the optimization if the stack lock is inside the stack
1299     // of another thread. The latter is avoided even in case of overflow
1300     // because we have guard pages at the end of all stacks. Hence, if
1301     // we go over the stack base and hit the stack of another thread,
1302     // this should not be in a writeable area that could contain a
1303     // stack lock allocated by that thread. As a consequence, a stack
1304     // lock less than page size away from SP is guaranteed to be
1305     // owned by the current thread.
1306     //
1307     // Note: assuming SP is aligned, we can check the low bits of
1308     // (mark-SP) instead of the low bits of mark. In that case,
1309     // assuming page size is a power of 2, we can merge the two
1310     // conditions into a single test:
1311     // => ((mark - SP) & (3 - os::pagesize())) == 0
1312 
1313 #ifdef AARCH64
1314     // Use the single check since the immediate is OK for AARCH64
1315     sub(R0, Rmark, Rstack_top);
1316     intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size());
1317     Assembler::LogicalImmediate imm(mask, false);
1318     ands(R0, R0, imm);
1319 
1320     // For recursive case store 0 into lock record.
1321     // It is harmless to store it unconditionally as lock record contains some garbage
1322     // value in its _displaced_header field by this moment.
1323     str(ZR, Address(Rlock, mark_offset));
1324 
1325 #else // AARCH64
1326     // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand.
1327     // Check independently the low bits and the distance to SP.
1328     // -1- test low 2 bits
1329     movs(R0, AsmOperand(Rmark, lsl, 30));
1330     // -2- test (mark - SP) if the low two bits are 0
1331     sub(R0, Rmark, SP, eq);
1332     movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq);
1333     // If still 'eq' then recursive locking OK: store 0 into lock record
1334     str(R0, Address(Rlock, mark_offset), eq);
1335 
1336 #endif // AARCH64
1337 
1338 #ifndef PRODUCT
1339     if (PrintBiasedLockingStatistics) {
1340       cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr());
1341     }
1342 #endif // !PRODUCT
1343 
1344     b(done, eq);
1345 
1346     bind(slow_case);
1347 
1348     // Call the runtime routine for slow case
1349     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
1350 
1351     bind(done);
1352   }
1353 }
1354 
1355 
1356 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
1357 //
1358 // Argument: R1: Points to BasicObjectLock structure for lock
1359 // Throw an IllegalMonitorException if object is not locked by current thread
1360 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM.
1361 void InterpreterMacroAssembler::unlock_object(Register Rlock) {
1362   assert(Rlock == R1, "the second argument");
1363 
1364   if (UseHeavyMonitors) {
1365     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
1366   } else {
1367     Label done, slow_case;
1368 
1369     const Register Robj = R2;
1370     const Register Rmark = R3;
1371     const Register Rresult = R0;
1372     assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp);
1373 
1374     const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1375     const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1376     const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
1377 
1378     const Register Rzero = zero_register(Rtemp);
1379 
1380     // Load oop into Robj
1381     ldr(Robj, Address(Rlock, obj_offset));
1382 
1383     // Free entry
1384     str(Rzero, Address(Rlock, obj_offset));
1385 
1386     if (UseBiasedLocking) {
1387       biased_locking_exit(Robj, Rmark, done);
1388     }
1389 
1390     // Load the old header from BasicLock structure
1391     ldr(Rmark, Address(Rlock, mark_offset));
1392 
1393     // Test for recursion (zero mark in BasicLock)
1394     cbz(Rmark, done);
1395 
1396     bool allow_fallthrough_on_failure = true;
1397 
1398     cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure);
1399 
1400     b(done, eq);
1401 
1402     bind(slow_case);
1403 
1404     // Call the runtime routine for slow case.
1405     str(Robj, Address(Rlock, obj_offset)); // restore obj
1406     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
1407 
1408     bind(done);
1409   }
1410 }
1411 
1412 
1413 // Test ImethodDataPtr.  If it is null, continue at the specified label
1414 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) {
1415   assert(ProfileInterpreter, "must be profiling interpreter");
1416   ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1417   cbz(mdp, zero_continue);
1418 }
1419 
1420 
1421 // Set the method data pointer for the current bcp.
1422 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR.
1423 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
1424   assert(ProfileInterpreter, "must be profiling interpreter");
1425   Label set_mdp;
1426 
1427   // Test MDO to avoid the call if it is NULL.
1428   ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
1429   cbz(Rtemp, set_mdp);
1430 
1431   mov(R0, Rmethod);
1432   mov(R1, Rbcp);
1433   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1);
1434   // R0/W0: mdi
1435 
1436   // mdo is guaranteed to be non-zero here, we checked for it before the call.
1437   ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
1438   add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()));
1439   add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0);
1440 
1441   bind(set_mdp);
1442   str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1443 }
1444 
1445 
1446 void InterpreterMacroAssembler::verify_method_data_pointer() {
1447   assert(ProfileInterpreter, "must be profiling interpreter");
1448 #ifdef ASSERT
1449   Label verify_continue;
1450   save_caller_save_registers();
1451 
1452   const Register Rmdp = R2;
1453   test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue
1454 
1455   // If the mdp is valid, it will point to a DataLayout header which is
1456   // consistent with the bcp.  The converse is highly probable also.
1457 
1458   ldrh(R3, Address(Rmdp, DataLayout::bci_offset()));
1459   ldr(Rtemp, Address(Rmethod, Method::const_offset()));
1460   add(R3, R3, Rtemp);
1461   add(R3, R3, in_bytes(ConstMethod::codes_offset()));
1462   cmp(R3, Rbcp);
1463   b(verify_continue, eq);
1464 
1465   mov(R0, Rmethod);
1466   mov(R1, Rbcp);
1467   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp);
1468 
1469   bind(verify_continue);
1470   restore_caller_save_registers();
1471 #endif // ASSERT
1472 }
1473 
1474 
1475 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) {
1476   assert(ProfileInterpreter, "must be profiling interpreter");
1477   assert_different_registers(mdp_in, value);
1478   str(value, Address(mdp_in, offset));
1479 }
1480 
1481 
1482 // Increments mdp data. Sets bumped_count register to adjusted counter.
1483 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
1484                                                       int offset,
1485                                                       Register bumped_count,
1486                                                       bool decrement) {
1487   assert(ProfileInterpreter, "must be profiling interpreter");
1488 
1489   // Counter address
1490   Address data(mdp_in, offset);
1491   assert_different_registers(mdp_in, bumped_count);
1492 
1493   increment_mdp_data_at(data, bumped_count, decrement);
1494 }
1495 
1496 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) {
1497   assert_different_registers(mdp_in, Rtemp);
1498   assert(ProfileInterpreter, "must be profiling interpreter");
1499   assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range");
1500 
1501   // Set the flag
1502   ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
1503   orr(Rtemp, Rtemp, (unsigned)flag_byte_constant);
1504   strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
1505 }
1506 
1507 
1508 // Increments mdp data. Sets bumped_count register to adjusted counter.
1509 void InterpreterMacroAssembler::increment_mdp_data_at(Address data,
1510                                                       Register bumped_count,
1511                                                       bool decrement) {
1512   assert(ProfileInterpreter, "must be profiling interpreter");
1513 
1514   ldr(bumped_count, data);
1515   if (decrement) {
1516     // Decrement the register. Set condition codes.
1517     subs(bumped_count, bumped_count, DataLayout::counter_increment);
1518     // Avoid overflow.
1519 #ifdef AARCH64
1520     assert(DataLayout::counter_increment == 1, "required for cinc");
1521     cinc(bumped_count, bumped_count, pl);
1522 #else
1523     add(bumped_count, bumped_count, DataLayout::counter_increment, pl);
1524 #endif // AARCH64
1525   } else {
1526     // Increment the register. Set condition codes.
1527     adds(bumped_count, bumped_count, DataLayout::counter_increment);
1528     // Avoid overflow.
1529 #ifdef AARCH64
1530     assert(DataLayout::counter_increment == 1, "required for cinv");
1531     cinv(bumped_count, bumped_count, mi); // inverts 0x80..00 back to 0x7f..ff
1532 #else
1533     sub(bumped_count, bumped_count, DataLayout::counter_increment, mi);
1534 #endif // AARCH64
1535   }
1536   str(bumped_count, data);
1537 }
1538 
1539 
1540 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
1541                                                  int offset,
1542                                                  Register value,
1543                                                  Register test_value_out,
1544                                                  Label& not_equal_continue) {
1545   assert(ProfileInterpreter, "must be profiling interpreter");
1546   assert_different_registers(mdp_in, test_value_out, value);
1547 
1548   ldr(test_value_out, Address(mdp_in, offset));
1549   cmp(test_value_out, value);
1550 
1551   b(not_equal_continue, ne);
1552 }
1553 
1554 
1555 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) {
1556   assert(ProfileInterpreter, "must be profiling interpreter");
1557   assert_different_registers(mdp_in, reg_temp);
1558 
1559   ldr(reg_temp, Address(mdp_in, offset_of_disp));
1560   add(mdp_in, mdp_in, reg_temp);
1561   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1562 }
1563 
1564 
1565 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) {
1566   assert(ProfileInterpreter, "must be profiling interpreter");
1567   assert_different_registers(mdp_in, reg_offset, reg_tmp);
1568 
1569   ldr(reg_tmp, Address(mdp_in, reg_offset));
1570   add(mdp_in, mdp_in, reg_tmp);
1571   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1572 }
1573 
1574 
1575 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) {
1576   assert(ProfileInterpreter, "must be profiling interpreter");
1577   add(mdp_in, mdp_in, constant);
1578   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1579 }
1580 
1581 
1582 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
1583 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
1584   assert(ProfileInterpreter, "must be profiling interpreter");
1585   assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp);
1586 
1587   mov(R1, return_bci);
1588   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1);
1589 }
1590 
1591 
1592 // Sets mdp, bumped_count registers, blows Rtemp.
1593 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) {
1594   assert_different_registers(mdp, bumped_count);
1595 
1596   if (ProfileInterpreter) {
1597     Label profile_continue;
1598 
1599     // If no method data exists, go to profile_continue.
1600     // Otherwise, assign to mdp
1601     test_method_data_pointer(mdp, profile_continue);
1602 
1603     // We are taking a branch. Increment the taken count.
1604     increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count);
1605 
1606     // The method data pointer needs to be updated to reflect the new target.
1607     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp);
1608 
1609     bind (profile_continue);
1610   }
1611 }
1612 
1613 
1614 // Sets mdp, blows Rtemp.
1615 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
1616   assert_different_registers(mdp, Rtemp);
1617 
1618   if (ProfileInterpreter) {
1619     Label profile_continue;
1620 
1621     // If no method data exists, go to profile_continue.
1622     test_method_data_pointer(mdp, profile_continue);
1623 
1624     // We are taking a branch.  Increment the not taken count.
1625     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp);
1626 
1627     // The method data pointer needs to be updated to correspond to the next bytecode
1628     update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
1629 
1630     bind (profile_continue);
1631   }
1632 }
1633 
1634 
1635 // Sets mdp, blows Rtemp.
1636 void InterpreterMacroAssembler::profile_call(Register mdp) {
1637   assert_different_registers(mdp, Rtemp);
1638 
1639   if (ProfileInterpreter) {
1640     Label profile_continue;
1641 
1642     // If no method data exists, go to profile_continue.
1643     test_method_data_pointer(mdp, profile_continue);
1644 
1645     // We are making a call.  Increment the count.
1646     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1647 
1648     // The method data pointer needs to be updated to reflect the new target.
1649     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1650 
1651     bind (profile_continue);
1652   }
1653 }
1654 
1655 
1656 // Sets mdp, blows Rtemp.
1657 void InterpreterMacroAssembler::profile_final_call(Register mdp) {
1658   if (ProfileInterpreter) {
1659     Label profile_continue;
1660 
1661     // If no method data exists, go to profile_continue.
1662     test_method_data_pointer(mdp, profile_continue);
1663 
1664     // We are making a call.  Increment the count.
1665     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1666 
1667     // The method data pointer needs to be updated to reflect the new target.
1668     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1669 
1670     bind (profile_continue);
1671   }
1672 }
1673 
1674 
1675 // Sets mdp, blows Rtemp.
1676 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) {
1677   assert_different_registers(mdp, receiver, Rtemp);
1678 
1679   if (ProfileInterpreter) {
1680     Label profile_continue;
1681 
1682     // If no method data exists, go to profile_continue.
1683     test_method_data_pointer(mdp, profile_continue);
1684 
1685     Label skip_receiver_profile;
1686     if (receiver_can_be_null) {
1687       Label not_null;
1688       cbnz(receiver, not_null);
1689       // We are making a call.  Increment the count for null receiver.
1690       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1691       b(skip_receiver_profile);
1692       bind(not_null);
1693     }
1694 
1695     // Record the receiver type.
1696     record_klass_in_profile(receiver, mdp, Rtemp, true);
1697     bind(skip_receiver_profile);
1698 
1699     // The method data pointer needs to be updated to reflect the new target.
1700     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1701     bind(profile_continue);
1702   }
1703 }
1704 
1705 
1706 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1707                                         Register receiver, Register mdp,
1708                                         Register reg_tmp,
1709                                         int start_row, Label& done, bool is_virtual_call) {
1710   if (TypeProfileWidth == 0)
1711     return;
1712 
1713   assert_different_registers(receiver, mdp, reg_tmp);
1714 
1715   int last_row = VirtualCallData::row_limit() - 1;
1716   assert(start_row <= last_row, "must be work left to do");
1717   // Test this row for both the receiver and for null.
1718   // Take any of three different outcomes:
1719   //   1. found receiver => increment count and goto done
1720   //   2. found null => keep looking for case 1, maybe allocate this cell
1721   //   3. found something else => keep looking for cases 1 and 2
1722   // Case 3 is handled by a recursive call.
1723   for (int row = start_row; row <= last_row; row++) {
1724     Label next_test;
1725 
1726     // See if the receiver is receiver[n].
1727     int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row));
1728 
1729     test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test);
1730 
1731     // The receiver is receiver[n].  Increment count[n].
1732     int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row));
1733     increment_mdp_data_at(mdp, count_offset, reg_tmp);
1734     b(done);
1735 
1736     bind(next_test);
1737     // reg_tmp now contains the receiver from the CallData.
1738 
1739     if (row == start_row) {
1740       Label found_null;
1741       // Failed the equality check on receiver[n]...  Test for null.
1742       if (start_row == last_row) {
1743         // The only thing left to do is handle the null case.
1744         if (is_virtual_call) {
1745           cbz(reg_tmp, found_null);
1746           // Receiver did not match any saved receiver and there is no empty row for it.
1747           // Increment total counter to indicate polymorphic case.
1748           increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp);
1749           b(done);
1750           bind(found_null);
1751         } else {
1752           cbnz(reg_tmp, done);
1753         }
1754         break;
1755       }
1756       // Since null is rare, make it be the branch-taken case.
1757       cbz(reg_tmp, found_null);
1758 
1759       // Put all the "Case 3" tests here.
1760       record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call);
1761 
1762       // Found a null.  Keep searching for a matching receiver,
1763       // but remember that this is an empty (unused) slot.
1764       bind(found_null);
1765     }
1766   }
1767 
1768   // In the fall-through case, we found no matching receiver, but we
1769   // observed the receiver[start_row] is NULL.
1770 
1771   // Fill in the receiver field and increment the count.
1772   int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row));
1773   set_mdp_data_at(mdp, recvr_offset, receiver);
1774   int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row));
1775   mov(reg_tmp, DataLayout::counter_increment);
1776   set_mdp_data_at(mdp, count_offset, reg_tmp);
1777   if (start_row > 0) {
1778     b(done);
1779   }
1780 }
1781 
1782 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1783                                                         Register mdp,
1784                                                         Register reg_tmp,
1785                                                         bool is_virtual_call) {
1786   assert(ProfileInterpreter, "must be profiling");
1787   assert_different_registers(receiver, mdp, reg_tmp);
1788 
1789   Label done;
1790 
1791   record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call);
1792 
1793   bind (done);
1794 }
1795 
1796 // Sets mdp, blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
1797 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) {
1798   assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3);
1799 
1800   if (ProfileInterpreter) {
1801     Label profile_continue;
1802     uint row;
1803 
1804     // If no method data exists, go to profile_continue.
1805     test_method_data_pointer(mdp, profile_continue);
1806 
1807     // Update the total ret count.
1808     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1809 
1810     for (row = 0; row < RetData::row_limit(); row++) {
1811       Label next_test;
1812 
1813       // See if return_bci is equal to bci[n]:
1814       test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci,
1815                        Rtemp, next_test);
1816 
1817       // return_bci is equal to bci[n].  Increment the count.
1818       increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp);
1819 
1820       // The method data pointer needs to be updated to reflect the new target.
1821       update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp);
1822       b(profile_continue);
1823       bind(next_test);
1824     }
1825 
1826     update_mdp_for_ret(return_bci);
1827 
1828     bind(profile_continue);
1829   }
1830 }
1831 
1832 
1833 // Sets mdp.
1834 void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
1835   if (ProfileInterpreter) {
1836     Label profile_continue;
1837 
1838     // If no method data exists, go to profile_continue.
1839     test_method_data_pointer(mdp, profile_continue);
1840 
1841     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1842 
1843     // The method data pointer needs to be updated.
1844     int mdp_delta = in_bytes(BitData::bit_data_size());
1845     if (TypeProfileCasts) {
1846       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1847     }
1848     update_mdp_by_constant(mdp, mdp_delta);
1849 
1850     bind (profile_continue);
1851   }
1852 }
1853 
1854 
1855 // Sets mdp, blows Rtemp.
1856 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) {
1857   assert_different_registers(mdp, Rtemp);
1858 
1859   if (ProfileInterpreter && TypeProfileCasts) {
1860     Label profile_continue;
1861 
1862     // If no method data exists, go to profile_continue.
1863     test_method_data_pointer(mdp, profile_continue);
1864 
1865     int count_offset = in_bytes(CounterData::count_offset());
1866     // Back up the address, since we have already bumped the mdp.
1867     count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
1868 
1869     // *Decrement* the counter.  We expect to see zero or small negatives.
1870     increment_mdp_data_at(mdp, count_offset, Rtemp, true);
1871 
1872     bind (profile_continue);
1873   }
1874 }
1875 
1876 
1877 // Sets mdp, blows Rtemp.
1878 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass)
1879 {
1880   assert_different_registers(mdp, klass, Rtemp);
1881 
1882   if (ProfileInterpreter) {
1883     Label profile_continue;
1884 
1885     // If no method data exists, go to profile_continue.
1886     test_method_data_pointer(mdp, profile_continue);
1887 
1888     // The method data pointer needs to be updated.
1889     int mdp_delta = in_bytes(BitData::bit_data_size());
1890     if (TypeProfileCasts) {
1891       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1892 
1893       // Record the object type.
1894       record_klass_in_profile(klass, mdp, Rtemp, false);
1895     }
1896     update_mdp_by_constant(mdp, mdp_delta);
1897 
1898     bind(profile_continue);
1899   }
1900 }
1901 
1902 
1903 // Sets mdp, blows Rtemp.
1904 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1905   assert_different_registers(mdp, Rtemp);
1906 
1907   if (ProfileInterpreter) {
1908     Label profile_continue;
1909 
1910     // If no method data exists, go to profile_continue.
1911     test_method_data_pointer(mdp, profile_continue);
1912 
1913     // Update the default case count
1914     increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp);
1915 
1916     // The method data pointer needs to be updated.
1917     update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp);
1918 
1919     bind(profile_continue);
1920   }
1921 }
1922 
1923 
1924 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2.
1925 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) {
1926   assert_different_registers(mdp, reg_tmp1, reg_tmp2);
1927   assert_different_registers(mdp, reg_tmp1, index);
1928 
1929   if (ProfileInterpreter) {
1930     Label profile_continue;
1931 
1932     const int count_offset = in_bytes(MultiBranchData::case_array_offset()) +
1933                               in_bytes(MultiBranchData::relative_count_offset());
1934 
1935     const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) +
1936                               in_bytes(MultiBranchData::relative_displacement_offset());
1937 
1938     // If no method data exists, go to profile_continue.
1939     test_method_data_pointer(mdp, profile_continue);
1940 
1941     // Build the base (index * per_case_size_in_bytes())
1942     logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size())));
1943 
1944     // Update the case count
1945     add(reg_tmp1, reg_tmp1, count_offset);
1946     increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2);
1947 
1948     // The method data pointer needs to be updated.
1949     add(reg_tmp1, reg_tmp1, displacement_offset - count_offset);
1950     update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2);
1951 
1952     bind (profile_continue);
1953   }
1954 }
1955 
1956 
1957 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) {
1958 #ifdef AARCH64
1959   rev_w(r, r);
1960 #else
1961   if (VM_Version::supports_rev()) {
1962     rev(r, r);
1963   } else {
1964     eor(rtmp1, r, AsmOperand(r, ror, 16));
1965     mvn(rtmp2, 0x0000ff00);
1966     andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8));
1967     eor(r, rtmp1, AsmOperand(r, ror, 8));
1968   }
1969 #endif // AARCH64
1970 }
1971 
1972 
1973 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) {
1974   const intx addr = (intx) (address_of_counter + offset);
1975 
1976   assert ((addr & 0x3) == 0, "address of counter should be aligned");
1977   const intx offset_mask = right_n_bits(AARCH64_ONLY(12 + 2) NOT_AARCH64(12));
1978 
1979   const address base = (address) (addr & ~offset_mask);
1980   const int offs = (int) (addr & offset_mask);
1981 
1982   const Register addr_base = tmp1;
1983   const Register val = tmp2;
1984 
1985   mov_slow(addr_base, base);
1986   ldr_s32(val, Address(addr_base, offs));
1987 
1988   if (avoid_overflow) {
1989     adds_32(val, val, 1);
1990 #ifdef AARCH64
1991     Label L;
1992     b(L, mi);
1993     str_32(val, Address(addr_base, offs));
1994     bind(L);
1995 #else
1996     str(val, Address(addr_base, offs), pl);
1997 #endif // AARCH64
1998   } else {
1999     add_32(val, val, 1);
2000     str_32(val, Address(addr_base, offs));
2001   }
2002 }
2003 
2004 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) {
2005   if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); }
2006 }
2007 
2008 // Inline assembly for:
2009 //
2010 // if (thread is in interp_only_mode) {
2011 //   InterpreterRuntime::post_method_entry();
2012 // }
2013 // if (DTraceMethodProbes) {
2014 //   SharedRuntime::dtrace_method_entry(method, receiver);
2015 // }
2016 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
2017 //   SharedRuntime::rc_trace_method_entry(method, receiver);
2018 // }
2019 
2020 void InterpreterMacroAssembler::notify_method_entry() {
2021   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
2022   // track stack depth.  If it is possible to enter interp_only_mode we add
2023   // the code to check if the event should be sent.
2024   if (can_post_interpreter_events()) {
2025     Label L;
2026 
2027     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
2028     cbz(Rtemp, L);
2029 
2030     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry));
2031 
2032     bind(L);
2033   }
2034 
2035   // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry
2036   if (DTraceMethodProbes) {
2037     Label Lcontinue;
2038 
2039     ldrb_global(Rtemp, (address)&DTraceMethodProbes);
2040     cbz(Rtemp, Lcontinue);
2041 
2042     mov(R0, Rthread);
2043     mov(R1, Rmethod);
2044     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1);
2045 
2046     bind(Lcontinue);
2047   }
2048   // RedefineClasses() tracing support for obsolete method entry
2049   if (log_is_enabled(Trace, redefine, class, obsolete)) {
2050     mov(R0, Rthread);
2051     mov(R1, Rmethod);
2052     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
2053                  R0, R1);
2054   }
2055 }
2056 
2057 
2058 void InterpreterMacroAssembler::notify_method_exit(
2059                  TosState state, NotifyMethodExitMode mode,
2060                  bool native, Register result_lo, Register result_hi, FloatRegister result_fp) {
2061   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
2062   // track stack depth.  If it is possible to enter interp_only_mode we add
2063   // the code to check if the event should be sent.
2064   if (mode == NotifyJVMTI && can_post_interpreter_events()) {
2065     Label L;
2066     // Note: frame::interpreter_frame_result has a dependency on how the
2067     // method result is saved across the call to post_method_exit. If this
2068     // is changed then the interpreter_frame_result implementation will
2069     // need to be updated too.
2070 
2071     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
2072     cbz(Rtemp, L);
2073 
2074     if (native) {
2075       // For c++ and template interpreter push both result registers on the
2076       // stack in native, we don't know the state.
2077       // On AArch64 result registers are stored into the frame at known locations.
2078       // See frame::interpreter_frame_result for code that gets the result values from here.
2079       assert(result_lo != noreg, "result registers should be defined");
2080 
2081 #ifdef AARCH64
2082       assert(result_hi == noreg, "result_hi is not used on AArch64");
2083       assert(result_fp != fnoreg, "FP result register must be defined");
2084 
2085       str_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize));
2086       str(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize));
2087 #else
2088       assert(result_hi != noreg, "result registers should be defined");
2089 
2090 #ifdef __ABI_HARD__
2091       assert(result_fp != fnoreg, "FP result register must be defined");
2092       sub(SP, SP, 2 * wordSize);
2093       fstd(result_fp, Address(SP));
2094 #endif // __ABI_HARD__
2095 
2096       push(RegisterSet(result_lo) | RegisterSet(result_hi));
2097 #endif // AARCH64
2098 
2099       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
2100 
2101 #ifdef AARCH64
2102       ldr_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize));
2103       ldr(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize));
2104 #else
2105       pop(RegisterSet(result_lo) | RegisterSet(result_hi));
2106 #ifdef __ABI_HARD__
2107       fldd(result_fp, Address(SP));
2108       add(SP, SP, 2 * wordSize);
2109 #endif // __ABI_HARD__
2110 #endif // AARCH64
2111 
2112     } else {
2113       // For the template interpreter, the value on tos is the size of the
2114       // state. (c++ interpreter calls jvmti somewhere else).
2115       push(state);
2116       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
2117       pop(state);
2118     }
2119 
2120     bind(L);
2121   }
2122 
2123   // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit
2124   if (DTraceMethodProbes) {
2125     Label Lcontinue;
2126 
2127     ldrb_global(Rtemp, (address)&DTraceMethodProbes);
2128     cbz(Rtemp, Lcontinue);
2129 
2130     push(state);
2131 
2132     mov(R0, Rthread);
2133     mov(R1, Rmethod);
2134 
2135     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1);
2136 
2137     pop(state);
2138 
2139     bind(Lcontinue);
2140   }
2141 }
2142 
2143 
2144 #ifndef PRODUCT
2145 
2146 void InterpreterMacroAssembler::trace_state(const char* msg) {
2147   int push_size = save_caller_save_registers();
2148 
2149   Label Lcontinue;
2150   InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n");
2151   InlinedString Lmsg(msg);
2152   InlinedAddress Lprintf((address)printf);
2153 
2154   ldr_literal(R0, Lmsg0);
2155   ldr_literal(R1, Lmsg);
2156   mov(R2, FP);
2157   add(R3, SP, push_size);  // original SP (without saved registers)
2158   ldr_literal(Rtemp, Lprintf);
2159   call(Rtemp);
2160 
2161   b(Lcontinue);
2162 
2163   bind_literal(Lmsg0);
2164   bind_literal(Lmsg);
2165   bind_literal(Lprintf);
2166 
2167 
2168   bind(Lcontinue);
2169 
2170   restore_caller_save_registers();
2171 }
2172 
2173 #endif
2174 
2175 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
2176 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
2177                                                         int increment, Address mask_addr,
2178                                                         Register scratch, Register scratch2,
2179                                                         AsmCondition cond, Label* where) {
2180   // caution: scratch2 and base address of counter_addr can be the same
2181   assert_different_registers(scratch, scratch2);
2182   ldr_u32(scratch, counter_addr);
2183   add(scratch, scratch, increment);
2184   str_32(scratch, counter_addr);
2185 
2186 #ifdef AARCH64
2187   ldr_u32(scratch2, mask_addr);
2188   ands_w(ZR, scratch, scratch2);
2189 #else
2190   ldr(scratch2, mask_addr);
2191   andrs(scratch, scratch, scratch2);
2192 #endif // AARCH64
2193   b(*where, cond);
2194 }
2195 
2196 void InterpreterMacroAssembler::get_method_counters(Register method,
2197                                                     Register Rcounters,
2198                                                     Label& skip) {
2199   const Address method_counters(method, Method::method_counters_offset());
2200   Label has_counters;
2201 
2202   ldr(Rcounters, method_counters);
2203   cbnz(Rcounters, has_counters);
2204 
2205 #ifdef AARCH64
2206   const Register tmp = Rcounters;
2207   const int saved_regs_size = 20*wordSize;
2208 
2209   // Note: call_VM will cut SP according to Rstack_top value before call, and restore SP to
2210   // extended_sp value from frame after the call.
2211   // So make sure there is enough stack space to save registers and adjust Rstack_top accordingly.
2212   {
2213     Label enough_stack_space;
2214     check_extended_sp(tmp);
2215     sub(Rstack_top, Rstack_top, saved_regs_size);
2216     cmp(SP, Rstack_top);
2217     b(enough_stack_space, ls);
2218 
2219     align_reg(tmp, Rstack_top, StackAlignmentInBytes);
2220     mov(SP, tmp);
2221     str(tmp, Address(FP, frame::interpreter_frame_extended_sp_offset * wordSize));
2222 
2223     bind(enough_stack_space);
2224     check_stack_top();
2225 
2226     int offset = 0;
2227     stp(R0,  R1,  Address(Rstack_top, offset)); offset += 2*wordSize;
2228     stp(R2,  R3,  Address(Rstack_top, offset)); offset += 2*wordSize;
2229     stp(R4,  R5,  Address(Rstack_top, offset)); offset += 2*wordSize;
2230     stp(R6,  R7,  Address(Rstack_top, offset)); offset += 2*wordSize;
2231     stp(R8,  R9,  Address(Rstack_top, offset)); offset += 2*wordSize;
2232     stp(R10, R11, Address(Rstack_top, offset)); offset += 2*wordSize;
2233     stp(R12, R13, Address(Rstack_top, offset)); offset += 2*wordSize;
2234     stp(R14, R15, Address(Rstack_top, offset)); offset += 2*wordSize;
2235     stp(R16, R17, Address(Rstack_top, offset)); offset += 2*wordSize;
2236     stp(R18, LR,  Address(Rstack_top, offset)); offset += 2*wordSize;
2237     assert (offset == saved_regs_size, "should be");
2238   }
2239 #else
2240   push(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(R14));
2241 #endif // AARCH64
2242 
2243   mov(R1, method);
2244   call_VM(noreg, CAST_FROM_FN_PTR(address,
2245           InterpreterRuntime::build_method_counters), R1);
2246 
2247 #ifdef AARCH64
2248   {
2249     int offset = 0;
2250     ldp(R0,  R1,  Address(Rstack_top, offset)); offset += 2*wordSize;
2251     ldp(R2,  R3,  Address(Rstack_top, offset)); offset += 2*wordSize;
2252     ldp(R4,  R5,  Address(Rstack_top, offset)); offset += 2*wordSize;
2253     ldp(R6,  R7,  Address(Rstack_top, offset)); offset += 2*wordSize;
2254     ldp(R8,  R9,  Address(Rstack_top, offset)); offset += 2*wordSize;
2255     ldp(R10, R11, Address(Rstack_top, offset)); offset += 2*wordSize;
2256     ldp(R12, R13, Address(Rstack_top, offset)); offset += 2*wordSize;
2257     ldp(R14, R15, Address(Rstack_top, offset)); offset += 2*wordSize;
2258     ldp(R16, R17, Address(Rstack_top, offset)); offset += 2*wordSize;
2259     ldp(R18, LR,  Address(Rstack_top, offset)); offset += 2*wordSize;
2260     assert (offset == saved_regs_size, "should be");
2261 
2262     add(Rstack_top, Rstack_top, saved_regs_size);
2263   }
2264 #else
2265   pop(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(R14));
2266 #endif // AARCH64
2267 
2268   ldr(Rcounters, method_counters);
2269   cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory
2270 
2271   bind(has_counters);
2272 }