1 /* 2 * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "jvm.h" 27 #include "gc/shared/barrierSet.inline.hpp" 28 #include "gc/shared/cardTable.hpp" 29 #include "gc/shared/cardTableBarrierSet.inline.hpp" 30 #include "gc/shared/collectedHeap.hpp" 31 #include "interp_masm_arm.hpp" 32 #include "interpreter/interpreter.hpp" 33 #include "interpreter/interpreterRuntime.hpp" 34 #include "logging/log.hpp" 35 #include "oops/arrayOop.hpp" 36 #include "oops/markOop.hpp" 37 #include "oops/method.hpp" 38 #include "oops/methodData.hpp" 39 #include "prims/jvmtiExport.hpp" 40 #include "prims/jvmtiThreadState.hpp" 41 #include "runtime/basicLock.hpp" 42 #include "runtime/biasedLocking.hpp" 43 #include "runtime/frame.inline.hpp" 44 #include "runtime/sharedRuntime.hpp" 45 46 #if INCLUDE_ALL_GCS 47 #include "gc/g1/g1BarrierSet.hpp" 48 #include "gc/g1/g1CollectedHeap.inline.hpp" 49 #include "gc/g1/heapRegion.hpp" 50 #endif // INCLUDE_ALL_GCS 51 52 //-------------------------------------------------------------------- 53 // Implementation of InterpreterMacroAssembler 54 55 56 57 58 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) { 59 } 60 61 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 62 #if defined(ASSERT) && !defined(AARCH64) 63 // Ensure that last_sp is not filled. 64 { Label L; 65 ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 66 cbz(Rtemp, L); 67 stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL"); 68 bind(L); 69 } 70 #endif // ASSERT && !AARCH64 71 72 // Rbcp must be saved/restored since it may change due to GC. 73 save_bcp(); 74 75 #ifdef AARCH64 76 check_no_cached_stack_top(Rtemp); 77 save_stack_top(); 78 check_extended_sp(Rtemp); 79 cut_sp_before_call(); 80 #endif // AARCH64 81 82 // super call 83 MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions); 84 85 #ifdef AARCH64 86 // Restore SP to extended SP 87 restore_sp_after_call(Rtemp); 88 check_stack_top(); 89 clear_cached_stack_top(); 90 #endif // AARCH64 91 92 // Restore interpreter specific registers. 93 restore_bcp(); 94 restore_method(); 95 } 96 97 void InterpreterMacroAssembler::jump_to_entry(address entry) { 98 assert(entry, "Entry must have been generated by now"); 99 b(entry); 100 } 101 102 void InterpreterMacroAssembler::check_and_handle_popframe() { 103 if (can_pop_frame()) { 104 Label L; 105 const Register popframe_cond = R2_tmp; 106 107 // Initiate popframe handling only if it is not already being processed. If the flag 108 // has the popframe_processing bit set, it means that this code is called *during* popframe 109 // handling - we don't want to reenter. 110 111 ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset())); 112 tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L); 113 tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L); 114 115 // Call Interpreter::remove_activation_preserving_args_entry() to get the 116 // address of the same-named entrypoint in the generated interpreter code. 117 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry)); 118 119 // Call indirectly to avoid generation ordering problem. 120 jump(R0); 121 122 bind(L); 123 } 124 } 125 126 127 // Blows R2, Rtemp. Sets TOS cached value. 128 void InterpreterMacroAssembler::load_earlyret_value(TosState state) { 129 const Register thread_state = R2_tmp; 130 131 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 132 133 const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset()); 134 const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset()); 135 const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset()); 136 #ifndef AARCH64 137 const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset() 138 + in_ByteSize(wordSize)); 139 #endif // !AARCH64 140 141 Register zero = zero_register(Rtemp); 142 143 switch (state) { 144 case atos: ldr(R0_tos, oop_addr); 145 str(zero, oop_addr); 146 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 147 break; 148 149 #ifdef AARCH64 150 case ltos: ldr(R0_tos, val_addr); break; 151 #else 152 case ltos: ldr(R1_tos_hi, val_addr_hi); // fall through 153 #endif // AARCH64 154 case btos: // fall through 155 case ztos: // fall through 156 case ctos: // fall through 157 case stos: // fall through 158 case itos: ldr_s32(R0_tos, val_addr); break; 159 #ifdef __SOFTFP__ 160 case dtos: ldr(R1_tos_hi, val_addr_hi); // fall through 161 case ftos: ldr(R0_tos, val_addr); break; 162 #else 163 case ftos: ldr_float (S0_tos, val_addr); break; 164 case dtos: ldr_double(D0_tos, val_addr); break; 165 #endif // __SOFTFP__ 166 case vtos: /* nothing to do */ break; 167 default : ShouldNotReachHere(); 168 } 169 // Clean up tos value in the thread object 170 str(zero, val_addr); 171 #ifndef AARCH64 172 str(zero, val_addr_hi); 173 #endif // !AARCH64 174 175 mov(Rtemp, (int) ilgl); 176 str_32(Rtemp, tos_addr); 177 } 178 179 180 // Blows R2, Rtemp. 181 void InterpreterMacroAssembler::check_and_handle_earlyret() { 182 if (can_force_early_return()) { 183 Label L; 184 const Register thread_state = R2_tmp; 185 186 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 187 cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit; 188 189 // Initiate earlyret handling only if it is not already being processed. 190 // If the flag has the earlyret_processing bit set, it means that this code 191 // is called *during* earlyret handling - we don't want to reenter. 192 193 ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset())); 194 cmp(Rtemp, JvmtiThreadState::earlyret_pending); 195 b(L, ne); 196 197 // Call Interpreter::remove_activation_early_entry() to get the address of the 198 // same-named entrypoint in the generated interpreter code. 199 200 ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset())); 201 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0); 202 203 jump(R0); 204 205 bind(L); 206 } 207 } 208 209 210 // Sets reg. Blows Rtemp. 211 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) { 212 assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode"); 213 assert(reg != Rtemp, "should be different registers"); 214 215 ldrb(Rtemp, Address(Rbcp, bcp_offset)); 216 ldrb(reg, Address(Rbcp, bcp_offset+1)); 217 orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte)); 218 } 219 220 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) { 221 assert_different_registers(index, tmp_reg); 222 if (index_size == sizeof(u2)) { 223 // load bytes of index separately to avoid unaligned access 224 ldrb(index, Address(Rbcp, bcp_offset+1)); 225 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 226 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 227 } else if (index_size == sizeof(u4)) { 228 // TODO-AARCH64: consider using unaligned access here 229 ldrb(index, Address(Rbcp, bcp_offset+3)); 230 ldrb(tmp_reg, Address(Rbcp, bcp_offset+2)); 231 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 232 ldrb(tmp_reg, Address(Rbcp, bcp_offset+1)); 233 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 234 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 235 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 236 // Check if the secondary index definition is still ~x, otherwise 237 // we have to change the following assembler code to calculate the 238 // plain index. 239 assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line"); 240 mvn_32(index, index); // convert to plain index 241 } else if (index_size == sizeof(u1)) { 242 ldrb(index, Address(Rbcp, bcp_offset)); 243 } else { 244 ShouldNotReachHere(); 245 } 246 } 247 248 // Sets cache, index. 249 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) { 250 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 251 assert_different_registers(cache, index); 252 253 get_index_at_bcp(index, bcp_offset, cache, index_size); 254 255 // load constant pool cache pointer 256 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 257 258 // convert from field index to ConstantPoolCacheEntry index 259 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 260 // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called 261 logical_shift_left(index, index, 2); 262 } 263 264 // Sets cache, index, bytecode. 265 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) { 266 get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size); 267 // caution index and bytecode can be the same 268 add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord)); 269 #ifdef AARCH64 270 add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())); 271 ldarb(bytecode, bytecode); 272 #else 273 ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()))); 274 TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true); 275 #endif // AARCH64 276 } 277 278 // Sets cache. Blows reg_tmp. 279 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) { 280 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 281 assert_different_registers(cache, reg_tmp); 282 283 get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size); 284 285 // load constant pool cache pointer 286 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 287 288 // skip past the header 289 add(cache, cache, in_bytes(ConstantPoolCache::base_offset())); 290 // convert from field index to ConstantPoolCacheEntry index 291 // and from word offset to byte offset 292 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 293 add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord)); 294 } 295 296 // Load object from cpool->resolved_references(index) 297 void InterpreterMacroAssembler::load_resolved_reference_at_index( 298 Register result, Register index) { 299 assert_different_registers(result, index); 300 get_constant_pool(result); 301 302 Register cache = result; 303 // load pointer for resolved_references[] objArray 304 ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes())); 305 ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes())); 306 resolve_oop_handle(cache); 307 // Add in the index 308 // convert from field index to resolved_references() index and from 309 // word index to byte offset. Since this is a java object, it can be compressed 310 add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop)); 311 load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT))); 312 } 313 314 void InterpreterMacroAssembler::load_resolved_klass_at_offset( 315 Register Rcpool, Register Rindex, Register Rklass) { 316 add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord)); 317 ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index 318 ldr(Rklass, Address(Rcpool, ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses 319 add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord)); 320 ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes())); 321 } 322 323 // Generate a subtype check: branch to not_subtype if sub_klass is 324 // not a subtype of super_klass. 325 // Profiling code for the subtype check failure (profile_typecheck_failed) 326 // should be explicitly generated by the caller in the not_subtype case. 327 // Blows Rtemp, tmp1, tmp2. 328 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, 329 Register Rsuper_klass, 330 Label ¬_subtype, 331 Register tmp1, 332 Register tmp2) { 333 334 assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp); 335 Label ok_is_subtype, loop, update_cache; 336 337 const Register super_check_offset = tmp1; 338 const Register cached_super = tmp2; 339 340 // Profile the not-null value's klass. 341 profile_typecheck(tmp1, Rsub_klass); 342 343 // Load the super-klass's check offset into 344 ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset())); 345 346 // Check for self 347 cmp(Rsub_klass, Rsuper_klass); 348 349 // Load from the sub-klass's super-class display list, or a 1-word cache of 350 // the secondary superclass list, or a failing value with a sentinel offset 351 // if the super-klass is an interface or exceptionally deep in the Java 352 // hierarchy and we have to scan the secondary superclass list the hard way. 353 // See if we get an immediate positive hit 354 ldr(cached_super, Address(Rsub_klass, super_check_offset)); 355 356 cond_cmp(Rsuper_klass, cached_super, ne); 357 b(ok_is_subtype, eq); 358 359 // Check for immediate negative hit 360 cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset())); 361 b(not_subtype, ne); 362 363 // Now do a linear scan of the secondary super-klass chain. 364 const Register supers_arr = tmp1; 365 const Register supers_cnt = tmp2; 366 const Register cur_super = Rtemp; 367 368 // Load objArrayOop of secondary supers. 369 ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset())); 370 371 ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length 372 #ifdef AARCH64 373 cbz(supers_cnt, not_subtype); 374 add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes()); 375 #else 376 cmp(supers_cnt, 0); 377 378 // Skip to the start of array elements and prefetch the first super-klass. 379 ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne); 380 b(not_subtype, eq); 381 #endif // AARCH64 382 383 bind(loop); 384 385 #ifdef AARCH64 386 ldr(cur_super, Address(supers_arr, wordSize, post_indexed)); 387 #endif // AARCH64 388 389 cmp(cur_super, Rsuper_klass); 390 b(update_cache, eq); 391 392 subs(supers_cnt, supers_cnt, 1); 393 394 #ifndef AARCH64 395 ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne); 396 #endif // !AARCH64 397 398 b(loop, ne); 399 400 b(not_subtype); 401 402 bind(update_cache); 403 // Must be equal but missed in cache. Update cache. 404 str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset())); 405 406 bind(ok_is_subtype); 407 } 408 409 410 // The 1st part of the store check. 411 // Sets card_table_base register. 412 void InterpreterMacroAssembler::store_check_part1(Register card_table_base) { 413 // Check barrier set type (should be card table) and element size 414 BarrierSet* bs = Universe::heap()->barrier_set(); 415 assert(bs->kind() == BarrierSet::CardTableBarrierSet, 416 "Wrong barrier set kind"); 417 418 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 419 CardTable* ct = ctbs->card_table(); 420 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "Adjust store check code"); 421 422 // Load card table base address. 423 424 /* Performance note. 425 426 There is an alternative way of loading card table base address 427 from thread descriptor, which may look more efficient: 428 429 ldr(card_table_base, Address(Rthread, JavaThread::card_table_base_offset())); 430 431 However, performance measurements of micro benchmarks and specJVM98 432 showed that loading of card table base from thread descriptor is 433 7-18% slower compared to loading of literal embedded into the code. 434 Possible cause is a cache miss (card table base address resides in a 435 rarely accessed area of thread descriptor). 436 */ 437 // TODO-AARCH64 Investigate if mov_slow is faster than ldr from Rthread on AArch64 438 mov_address(card_table_base, (address)ct->byte_map_base(), symbolic_Relocation::card_table_reference); 439 } 440 441 // The 2nd part of the store check. 442 void InterpreterMacroAssembler::store_check_part2(Register obj, Register card_table_base, Register tmp) { 443 assert_different_registers(obj, card_table_base, tmp); 444 445 assert(CardTable::dirty_card_val() == 0, "Dirty card value must be 0 due to optimizations."); 446 #ifdef AARCH64 447 add(card_table_base, card_table_base, AsmOperand(obj, lsr, CardTable::card_shift)); 448 Address card_table_addr(card_table_base); 449 #else 450 Address card_table_addr(card_table_base, obj, lsr, CardTable::card_shift); 451 #endif 452 453 if (UseCondCardMark) { 454 if (UseConcMarkSweepGC) { 455 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), noreg); 456 } 457 Label already_dirty; 458 459 ldrb(tmp, card_table_addr); 460 cbz(tmp, already_dirty); 461 462 set_card(card_table_base, card_table_addr, tmp); 463 bind(already_dirty); 464 465 } else { 466 if (UseConcMarkSweepGC && CMSPrecleaningEnabled) { 467 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg); 468 } 469 set_card(card_table_base, card_table_addr, tmp); 470 } 471 } 472 473 void InterpreterMacroAssembler::set_card(Register card_table_base, Address card_table_addr, Register tmp) { 474 #ifdef AARCH64 475 strb(ZR, card_table_addr); 476 #else 477 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(Universe::heap()->barrier_set()); 478 CardTable* ct = ctbs->card_table(); 479 if ((((uintptr_t)ct->byte_map_base() & 0xff) == 0)) { 480 // Card table is aligned so the lowest byte of the table address base is zero. 481 // This works only if the code is not saved for later use, possibly 482 // in a context where the base would no longer be aligned. 483 strb(card_table_base, card_table_addr); 484 } else { 485 mov(tmp, 0); 486 strb(tmp, card_table_addr); 487 } 488 #endif // AARCH64 489 } 490 491 ////////////////////////////////////////////////////////////////////////////////// 492 493 494 // Java Expression Stack 495 496 void InterpreterMacroAssembler::pop_ptr(Register r) { 497 assert(r != Rstack_top, "unpredictable instruction"); 498 ldr(r, Address(Rstack_top, wordSize, post_indexed)); 499 } 500 501 void InterpreterMacroAssembler::pop_i(Register r) { 502 assert(r != Rstack_top, "unpredictable instruction"); 503 ldr_s32(r, Address(Rstack_top, wordSize, post_indexed)); 504 zap_high_non_significant_bits(r); 505 } 506 507 #ifdef AARCH64 508 void InterpreterMacroAssembler::pop_l(Register r) { 509 assert(r != Rstack_top, "unpredictable instruction"); 510 ldr(r, Address(Rstack_top, 2*wordSize, post_indexed)); 511 } 512 #else 513 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) { 514 assert_different_registers(lo, hi); 515 assert(lo < hi, "lo must be < hi"); 516 pop(RegisterSet(lo) | RegisterSet(hi)); 517 } 518 #endif // AARCH64 519 520 void InterpreterMacroAssembler::pop_f(FloatRegister fd) { 521 #ifdef AARCH64 522 ldr_s(fd, Address(Rstack_top, wordSize, post_indexed)); 523 #else 524 fpops(fd); 525 #endif // AARCH64 526 } 527 528 void InterpreterMacroAssembler::pop_d(FloatRegister fd) { 529 #ifdef AARCH64 530 ldr_d(fd, Address(Rstack_top, 2*wordSize, post_indexed)); 531 #else 532 fpopd(fd); 533 #endif // AARCH64 534 } 535 536 537 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value. 538 void InterpreterMacroAssembler::pop(TosState state) { 539 switch (state) { 540 case atos: pop_ptr(R0_tos); break; 541 case btos: // fall through 542 case ztos: // fall through 543 case ctos: // fall through 544 case stos: // fall through 545 case itos: pop_i(R0_tos); break; 546 #ifdef AARCH64 547 case ltos: pop_l(R0_tos); break; 548 #else 549 case ltos: pop_l(R0_tos_lo, R1_tos_hi); break; 550 #endif // AARCH64 551 #ifdef __SOFTFP__ 552 case ftos: pop_i(R0_tos); break; 553 case dtos: pop_l(R0_tos_lo, R1_tos_hi); break; 554 #else 555 case ftos: pop_f(S0_tos); break; 556 case dtos: pop_d(D0_tos); break; 557 #endif // __SOFTFP__ 558 case vtos: /* nothing to do */ break; 559 default : ShouldNotReachHere(); 560 } 561 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 562 } 563 564 void InterpreterMacroAssembler::push_ptr(Register r) { 565 assert(r != Rstack_top, "unpredictable instruction"); 566 str(r, Address(Rstack_top, -wordSize, pre_indexed)); 567 check_stack_top_on_expansion(); 568 } 569 570 void InterpreterMacroAssembler::push_i(Register r) { 571 assert(r != Rstack_top, "unpredictable instruction"); 572 str_32(r, Address(Rstack_top, -wordSize, pre_indexed)); 573 check_stack_top_on_expansion(); 574 } 575 576 #ifdef AARCH64 577 void InterpreterMacroAssembler::push_l(Register r) { 578 assert(r != Rstack_top, "unpredictable instruction"); 579 stp(r, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 580 check_stack_top_on_expansion(); 581 } 582 #else 583 void InterpreterMacroAssembler::push_l(Register lo, Register hi) { 584 assert_different_registers(lo, hi); 585 assert(lo < hi, "lo must be < hi"); 586 push(RegisterSet(lo) | RegisterSet(hi)); 587 } 588 #endif // AARCH64 589 590 void InterpreterMacroAssembler::push_f() { 591 #ifdef AARCH64 592 str_s(S0_tos, Address(Rstack_top, -wordSize, pre_indexed)); 593 check_stack_top_on_expansion(); 594 #else 595 fpushs(S0_tos); 596 #endif // AARCH64 597 } 598 599 void InterpreterMacroAssembler::push_d() { 600 #ifdef AARCH64 601 str_d(D0_tos, Address(Rstack_top, -2*wordSize, pre_indexed)); 602 check_stack_top_on_expansion(); 603 #else 604 fpushd(D0_tos); 605 #endif // AARCH64 606 } 607 608 // Transition state -> vtos. Blows Rtemp. 609 void InterpreterMacroAssembler::push(TosState state) { 610 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 611 switch (state) { 612 case atos: push_ptr(R0_tos); break; 613 case btos: // fall through 614 case ztos: // fall through 615 case ctos: // fall through 616 case stos: // fall through 617 case itos: push_i(R0_tos); break; 618 #ifdef AARCH64 619 case ltos: push_l(R0_tos); break; 620 #else 621 case ltos: push_l(R0_tos_lo, R1_tos_hi); break; 622 #endif // AARCH64 623 #ifdef __SOFTFP__ 624 case ftos: push_i(R0_tos); break; 625 case dtos: push_l(R0_tos_lo, R1_tos_hi); break; 626 #else 627 case ftos: push_f(); break; 628 case dtos: push_d(); break; 629 #endif // __SOFTFP__ 630 case vtos: /* nothing to do */ break; 631 default : ShouldNotReachHere(); 632 } 633 } 634 635 636 #ifndef AARCH64 637 638 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value. 639 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) { 640 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 641 // According to interpreter calling conventions, result is returned in R0/R1, 642 // but templates expect ftos in S0, and dtos in D0. 643 if (state == ftos) { 644 fmsr(S0_tos, R0); 645 } else if (state == dtos) { 646 fmdrr(D0_tos, R0, R1); 647 } 648 #endif // !__SOFTFP__ && !__ABI_HARD__ 649 } 650 651 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions). 652 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) { 653 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 654 // According to interpreter calling conventions, result is returned in R0/R1, 655 // so ftos (S0) and dtos (D0) are moved to R0/R1. 656 if (state == ftos) { 657 fmrs(R0, S0_tos); 658 } else if (state == dtos) { 659 fmrrd(R0, R1, D0_tos); 660 } 661 #endif // !__SOFTFP__ && !__ABI_HARD__ 662 } 663 664 #endif // !AARCH64 665 666 667 // Helpers for swap and dup 668 void InterpreterMacroAssembler::load_ptr(int n, Register val) { 669 ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 670 } 671 672 void InterpreterMacroAssembler::store_ptr(int n, Register val) { 673 str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 674 } 675 676 677 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() { 678 #ifdef AARCH64 679 check_no_cached_stack_top(Rtemp); 680 save_stack_top(); 681 cut_sp_before_call(); 682 mov(Rparams, Rstack_top); 683 #endif // AARCH64 684 685 // set sender sp 686 mov(Rsender_sp, SP); 687 688 #ifndef AARCH64 689 // record last_sp 690 str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 691 #endif // !AARCH64 692 } 693 694 // Jump to from_interpreted entry of a call unless single stepping is possible 695 // in this thread in which case we must call the i2i entry 696 void InterpreterMacroAssembler::jump_from_interpreted(Register method) { 697 assert_different_registers(method, Rtemp); 698 699 prepare_to_jump_from_interpreted(); 700 701 if (can_post_interpreter_events()) { 702 // JVMTI events, such as single-stepping, are implemented partly by avoiding running 703 // compiled code in threads for which the event is enabled. Check here for 704 // interp_only_mode if these events CAN be enabled. 705 706 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 707 #ifdef AARCH64 708 { 709 Label not_interp_only_mode; 710 711 cbz(Rtemp, not_interp_only_mode); 712 indirect_jump(Address(method, Method::interpreter_entry_offset()), Rtemp); 713 714 bind(not_interp_only_mode); 715 } 716 #else 717 cmp(Rtemp, 0); 718 ldr(PC, Address(method, Method::interpreter_entry_offset()), ne); 719 #endif // AARCH64 720 } 721 722 indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp); 723 } 724 725 726 void InterpreterMacroAssembler::restore_dispatch() { 727 mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos)); 728 } 729 730 731 // The following two routines provide a hook so that an implementation 732 // can schedule the dispatch in two parts. 733 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) { 734 // Nothing ARM-specific to be done here. 735 } 736 737 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) { 738 dispatch_next(state, step); 739 } 740 741 void InterpreterMacroAssembler::dispatch_base(TosState state, 742 DispatchTableMode table_mode, 743 bool verifyoop) { 744 if (VerifyActivationFrameSize) { 745 Label L; 746 #ifdef AARCH64 747 mov(Rtemp, SP); 748 sub(Rtemp, FP, Rtemp); 749 #else 750 sub(Rtemp, FP, SP); 751 #endif // AARCH64 752 int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize; 753 cmp(Rtemp, min_frame_size); 754 b(L, ge); 755 stop("broken stack frame"); 756 bind(L); 757 } 758 759 if (verifyoop) { 760 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 761 } 762 763 if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) { 764 zap_high_non_significant_bits(R0_tos); 765 } 766 767 #ifdef ASSERT 768 Label L; 769 mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos)); 770 cmp(Rtemp, RdispatchTable); 771 b(L, eq); 772 stop("invalid RdispatchTable"); 773 bind(L); 774 #endif 775 776 if (table_mode == DispatchDefault) { 777 if (state == vtos) { 778 indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp); 779 } else { 780 #ifdef AARCH64 781 sub(Rtemp, R3_bytecode, (Interpreter::distance_from_dispatch_table(vtos) - 782 Interpreter::distance_from_dispatch_table(state))); 783 indirect_jump(Address::indexed_ptr(RdispatchTable, Rtemp), Rtemp); 784 #else 785 // on 32-bit ARM this method is faster than the one above. 786 sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) - 787 Interpreter::distance_from_dispatch_table(state)) * wordSize); 788 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 789 #endif 790 } 791 } else { 792 assert(table_mode == DispatchNormal, "invalid dispatch table mode"); 793 address table = (address) Interpreter::normal_table(state); 794 mov_slow(Rtemp, table); 795 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 796 } 797 798 nop(); // to avoid filling CPU pipeline with invalid instructions 799 nop(); 800 } 801 802 void InterpreterMacroAssembler::dispatch_only(TosState state) { 803 dispatch_base(state, DispatchDefault); 804 } 805 806 807 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) { 808 dispatch_base(state, DispatchNormal); 809 } 810 811 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) { 812 dispatch_base(state, DispatchNormal, false); 813 } 814 815 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) { 816 // load next bytecode and advance Rbcp 817 ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed)); 818 dispatch_base(state, DispatchDefault); 819 } 820 821 void InterpreterMacroAssembler::narrow(Register result) { 822 // mask integer result to narrower return type. 823 const Register Rtmp = R2; 824 825 // get method type 826 ldr(Rtmp, Address(Rmethod, Method::const_offset())); 827 ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset())); 828 829 Label notBool, notByte, notChar, done; 830 cmp(Rtmp, T_INT); 831 b(done, eq); 832 833 cmp(Rtmp, T_BOOLEAN); 834 b(notBool, ne); 835 and_32(result, result, 1); 836 b(done); 837 838 bind(notBool); 839 cmp(Rtmp, T_BYTE); 840 b(notByte, ne); 841 sign_extend(result, result, 8); 842 b(done); 843 844 bind(notByte); 845 cmp(Rtmp, T_CHAR); 846 b(notChar, ne); 847 zero_extend(result, result, 16); 848 b(done); 849 850 bind(notChar); 851 // cmp(Rtmp, T_SHORT); 852 // b(done, ne); 853 sign_extend(result, result, 16); 854 855 // Nothing to do 856 bind(done); 857 } 858 859 // remove activation 860 // 861 // Unlock the receiver if this is a synchronized method. 862 // Unlock any Java monitors from syncronized blocks. 863 // Remove the activation from the stack. 864 // 865 // If there are locked Java monitors 866 // If throw_monitor_exception 867 // throws IllegalMonitorStateException 868 // Else if install_monitor_exception 869 // installs IllegalMonitorStateException 870 // Else 871 // no error processing 872 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr, 873 bool throw_monitor_exception, 874 bool install_monitor_exception, 875 bool notify_jvmdi) { 876 Label unlock, unlocked, no_unlock; 877 878 // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result. 879 880 const Address do_not_unlock_if_synchronized(Rthread, 881 JavaThread::do_not_unlock_if_synchronized_offset()); 882 883 const Register Rflag = R2; 884 const Register Raccess_flags = R3; 885 886 restore_method(); 887 888 ldrb(Rflag, do_not_unlock_if_synchronized); 889 890 // get method access flags 891 ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset())); 892 893 strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag 894 895 // check if method is synchronized 896 897 tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked); 898 899 // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set. 900 cbnz(Rflag, no_unlock); 901 902 // unlock monitor 903 push(state); // save result 904 905 // BasicObjectLock will be first in list, since this is a synchronized method. However, need 906 // to check that the object has not been unlocked by an explicit monitorexit bytecode. 907 908 const Register Rmonitor = R1; // fixed in unlock_object() 909 const Register Robj = R2; 910 911 // address of first monitor 912 sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock)); 913 914 ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes())); 915 cbnz(Robj, unlock); 916 917 pop(state); 918 919 if (throw_monitor_exception) { 920 // Entry already unlocked, need to throw exception 921 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 922 should_not_reach_here(); 923 } else { 924 // Monitor already unlocked during a stack unroll. 925 // If requested, install an illegal_monitor_state_exception. 926 // Continue with stack unrolling. 927 if (install_monitor_exception) { 928 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 929 } 930 b(unlocked); 931 } 932 933 934 // Exception case for the check that all monitors are unlocked. 935 const Register Rcur = R2; 936 Label restart_check_monitors_unlocked, exception_monitor_is_still_locked; 937 938 bind(exception_monitor_is_still_locked); 939 // Monitor entry is still locked, need to throw exception. 940 // Rcur: monitor entry. 941 942 if (throw_monitor_exception) { 943 // Throw exception 944 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 945 should_not_reach_here(); 946 } else { 947 // Stack unrolling. Unlock object and install illegal_monitor_exception 948 // Unlock does not block, so don't have to worry about the frame 949 950 push(state); 951 mov(R1, Rcur); 952 unlock_object(R1); 953 954 if (install_monitor_exception) { 955 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 956 } 957 958 pop(state); 959 b(restart_check_monitors_unlocked); 960 } 961 962 bind(unlock); 963 unlock_object(Rmonitor); 964 pop(state); 965 966 // Check that for block-structured locking (i.e., that all locked objects has been unlocked) 967 bind(unlocked); 968 969 // Check that all monitors are unlocked 970 { 971 Label loop; 972 973 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; 974 const Register Rbottom = R3; 975 const Register Rcur_obj = Rtemp; 976 977 bind(restart_check_monitors_unlocked); 978 979 ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize)); 980 // points to current entry, starting with top-most entry 981 sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize); 982 // points to word before bottom of monitor block 983 984 cmp(Rcur, Rbottom); // check if there are no monitors 985 #ifndef AARCH64 986 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 987 // prefetch monitor's object 988 #endif // !AARCH64 989 b(no_unlock, eq); 990 991 bind(loop); 992 #ifdef AARCH64 993 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes())); 994 #endif // AARCH64 995 // check if current entry is used 996 cbnz(Rcur_obj, exception_monitor_is_still_locked); 997 998 add(Rcur, Rcur, entry_size); // otherwise advance to next entry 999 cmp(Rcur, Rbottom); // check if bottom reached 1000 #ifndef AARCH64 1001 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 1002 // prefetch monitor's object 1003 #endif // !AARCH64 1004 b(loop, ne); // if not at bottom then check this entry 1005 } 1006 1007 bind(no_unlock); 1008 1009 // jvmti support 1010 if (notify_jvmdi) { 1011 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA 1012 } else { 1013 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA 1014 } 1015 1016 // remove activation 1017 #ifdef AARCH64 1018 ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize)); 1019 ldp(FP, LR, Address(FP)); 1020 mov(SP, Rtemp); 1021 #else 1022 mov(Rtemp, FP); 1023 ldmia(FP, RegisterSet(FP) | RegisterSet(LR)); 1024 ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize)); 1025 #endif 1026 1027 if (ret_addr != LR) { 1028 mov(ret_addr, LR); 1029 } 1030 } 1031 1032 1033 // At certain points in the method invocation the monitor of 1034 // synchronized methods hasn't been entered yet. 1035 // To correctly handle exceptions at these points, we set the thread local 1036 // variable _do_not_unlock_if_synchronized to true. The remove_activation will 1037 // check this flag. 1038 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) { 1039 const Address do_not_unlock_if_synchronized(Rthread, 1040 JavaThread::do_not_unlock_if_synchronized_offset()); 1041 if (flag) { 1042 mov(tmp, 1); 1043 strb(tmp, do_not_unlock_if_synchronized); 1044 } else { 1045 strb(zero_register(tmp), do_not_unlock_if_synchronized); 1046 } 1047 } 1048 1049 // Lock object 1050 // 1051 // Argument: R1 : Points to BasicObjectLock to be used for locking. 1052 // Must be initialized with object to lock. 1053 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1054 void InterpreterMacroAssembler::lock_object(Register Rlock) { 1055 assert(Rlock == R1, "the second argument"); 1056 1057 if (UseHeavyMonitors) { 1058 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1059 } else { 1060 Label done; 1061 1062 const Register Robj = R2; 1063 const Register Rmark = R3; 1064 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1065 1066 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1067 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1068 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1069 1070 Label already_locked, slow_case; 1071 1072 // Load object pointer 1073 ldr(Robj, Address(Rlock, obj_offset)); 1074 1075 if (UseBiasedLocking) { 1076 biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case); 1077 } 1078 1079 #ifdef AARCH64 1080 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1081 ldr(Rmark, Robj); 1082 1083 // Test if object is already locked 1084 assert(markOopDesc::unlocked_value == 1, "adjust this code"); 1085 tbz(Rmark, exact_log2(markOopDesc::unlocked_value), already_locked); 1086 1087 #else // AARCH64 1088 1089 // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread. 1090 // That would be acceptable as ether CAS or slow case path is taken in that case. 1091 // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as 1092 // loads are satisfied from a store queue if performed on the same processor). 1093 1094 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1095 ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes())); 1096 1097 // Test if object is already locked 1098 tst(Rmark, markOopDesc::unlocked_value); 1099 b(already_locked, eq); 1100 1101 #endif // !AARCH64 1102 // Save old object->mark() into BasicLock's displaced header 1103 str(Rmark, Address(Rlock, mark_offset)); 1104 1105 cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case); 1106 1107 #ifndef PRODUCT 1108 if (PrintBiasedLockingStatistics) { 1109 cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr()); 1110 } 1111 #endif //!PRODUCT 1112 1113 b(done); 1114 1115 // If we got here that means the object is locked by ether calling thread or another thread. 1116 bind(already_locked); 1117 // Handling of locked objects: recursive locks and slow case. 1118 1119 // Fast check for recursive lock. 1120 // 1121 // Can apply the optimization only if this is a stack lock 1122 // allocated in this thread. For efficiency, we can focus on 1123 // recently allocated stack locks (instead of reading the stack 1124 // base and checking whether 'mark' points inside the current 1125 // thread stack): 1126 // 1) (mark & 3) == 0 1127 // 2) SP <= mark < SP + os::pagesize() 1128 // 1129 // Warning: SP + os::pagesize can overflow the stack base. We must 1130 // neither apply the optimization for an inflated lock allocated 1131 // just above the thread stack (this is why condition 1 matters) 1132 // nor apply the optimization if the stack lock is inside the stack 1133 // of another thread. The latter is avoided even in case of overflow 1134 // because we have guard pages at the end of all stacks. Hence, if 1135 // we go over the stack base and hit the stack of another thread, 1136 // this should not be in a writeable area that could contain a 1137 // stack lock allocated by that thread. As a consequence, a stack 1138 // lock less than page size away from SP is guaranteed to be 1139 // owned by the current thread. 1140 // 1141 // Note: assuming SP is aligned, we can check the low bits of 1142 // (mark-SP) instead of the low bits of mark. In that case, 1143 // assuming page size is a power of 2, we can merge the two 1144 // conditions into a single test: 1145 // => ((mark - SP) & (3 - os::pagesize())) == 0 1146 1147 #ifdef AARCH64 1148 // Use the single check since the immediate is OK for AARCH64 1149 sub(R0, Rmark, Rstack_top); 1150 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size()); 1151 Assembler::LogicalImmediate imm(mask, false); 1152 ands(R0, R0, imm); 1153 1154 // For recursive case store 0 into lock record. 1155 // It is harmless to store it unconditionally as lock record contains some garbage 1156 // value in its _displaced_header field by this moment. 1157 str(ZR, Address(Rlock, mark_offset)); 1158 1159 #else // AARCH64 1160 // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand. 1161 // Check independently the low bits and the distance to SP. 1162 // -1- test low 2 bits 1163 movs(R0, AsmOperand(Rmark, lsl, 30)); 1164 // -2- test (mark - SP) if the low two bits are 0 1165 sub(R0, Rmark, SP, eq); 1166 movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq); 1167 // If still 'eq' then recursive locking OK: store 0 into lock record 1168 str(R0, Address(Rlock, mark_offset), eq); 1169 1170 #endif // AARCH64 1171 1172 #ifndef PRODUCT 1173 if (PrintBiasedLockingStatistics) { 1174 cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr()); 1175 } 1176 #endif // !PRODUCT 1177 1178 b(done, eq); 1179 1180 bind(slow_case); 1181 1182 // Call the runtime routine for slow case 1183 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1184 1185 bind(done); 1186 } 1187 } 1188 1189 1190 // Unlocks an object. Used in monitorexit bytecode and remove_activation. 1191 // 1192 // Argument: R1: Points to BasicObjectLock structure for lock 1193 // Throw an IllegalMonitorException if object is not locked by current thread 1194 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1195 void InterpreterMacroAssembler::unlock_object(Register Rlock) { 1196 assert(Rlock == R1, "the second argument"); 1197 1198 if (UseHeavyMonitors) { 1199 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1200 } else { 1201 Label done, slow_case; 1202 1203 const Register Robj = R2; 1204 const Register Rmark = R3; 1205 const Register Rresult = R0; 1206 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1207 1208 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1209 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1210 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1211 1212 const Register Rzero = zero_register(Rtemp); 1213 1214 // Load oop into Robj 1215 ldr(Robj, Address(Rlock, obj_offset)); 1216 1217 // Free entry 1218 str(Rzero, Address(Rlock, obj_offset)); 1219 1220 if (UseBiasedLocking) { 1221 biased_locking_exit(Robj, Rmark, done); 1222 } 1223 1224 // Load the old header from BasicLock structure 1225 ldr(Rmark, Address(Rlock, mark_offset)); 1226 1227 // Test for recursion (zero mark in BasicLock) 1228 cbz(Rmark, done); 1229 1230 bool allow_fallthrough_on_failure = true; 1231 1232 cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure); 1233 1234 b(done, eq); 1235 1236 bind(slow_case); 1237 1238 // Call the runtime routine for slow case. 1239 str(Robj, Address(Rlock, obj_offset)); // restore obj 1240 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1241 1242 bind(done); 1243 } 1244 } 1245 1246 1247 // Test ImethodDataPtr. If it is null, continue at the specified label 1248 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) { 1249 assert(ProfileInterpreter, "must be profiling interpreter"); 1250 ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1251 cbz(mdp, zero_continue); 1252 } 1253 1254 1255 // Set the method data pointer for the current bcp. 1256 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. 1257 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { 1258 assert(ProfileInterpreter, "must be profiling interpreter"); 1259 Label set_mdp; 1260 1261 // Test MDO to avoid the call if it is NULL. 1262 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1263 cbz(Rtemp, set_mdp); 1264 1265 mov(R0, Rmethod); 1266 mov(R1, Rbcp); 1267 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1); 1268 // R0/W0: mdi 1269 1270 // mdo is guaranteed to be non-zero here, we checked for it before the call. 1271 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1272 add(Rtemp, Rtemp, in_bytes(MethodData::data_offset())); 1273 add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0); 1274 1275 bind(set_mdp); 1276 str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1277 } 1278 1279 1280 void InterpreterMacroAssembler::verify_method_data_pointer() { 1281 assert(ProfileInterpreter, "must be profiling interpreter"); 1282 #ifdef ASSERT 1283 Label verify_continue; 1284 save_caller_save_registers(); 1285 1286 const Register Rmdp = R2; 1287 test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue 1288 1289 // If the mdp is valid, it will point to a DataLayout header which is 1290 // consistent with the bcp. The converse is highly probable also. 1291 1292 ldrh(R3, Address(Rmdp, DataLayout::bci_offset())); 1293 ldr(Rtemp, Address(Rmethod, Method::const_offset())); 1294 add(R3, R3, Rtemp); 1295 add(R3, R3, in_bytes(ConstMethod::codes_offset())); 1296 cmp(R3, Rbcp); 1297 b(verify_continue, eq); 1298 1299 mov(R0, Rmethod); 1300 mov(R1, Rbcp); 1301 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp); 1302 1303 bind(verify_continue); 1304 restore_caller_save_registers(); 1305 #endif // ASSERT 1306 } 1307 1308 1309 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) { 1310 assert(ProfileInterpreter, "must be profiling interpreter"); 1311 assert_different_registers(mdp_in, value); 1312 str(value, Address(mdp_in, offset)); 1313 } 1314 1315 1316 // Increments mdp data. Sets bumped_count register to adjusted counter. 1317 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in, 1318 int offset, 1319 Register bumped_count, 1320 bool decrement) { 1321 assert(ProfileInterpreter, "must be profiling interpreter"); 1322 1323 // Counter address 1324 Address data(mdp_in, offset); 1325 assert_different_registers(mdp_in, bumped_count); 1326 1327 increment_mdp_data_at(data, bumped_count, decrement); 1328 } 1329 1330 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) { 1331 assert_different_registers(mdp_in, Rtemp); 1332 assert(ProfileInterpreter, "must be profiling interpreter"); 1333 assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range"); 1334 1335 // Set the flag 1336 ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1337 orr(Rtemp, Rtemp, (unsigned)flag_byte_constant); 1338 strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1339 } 1340 1341 1342 // Increments mdp data. Sets bumped_count register to adjusted counter. 1343 void InterpreterMacroAssembler::increment_mdp_data_at(Address data, 1344 Register bumped_count, 1345 bool decrement) { 1346 assert(ProfileInterpreter, "must be profiling interpreter"); 1347 1348 ldr(bumped_count, data); 1349 if (decrement) { 1350 // Decrement the register. Set condition codes. 1351 subs(bumped_count, bumped_count, DataLayout::counter_increment); 1352 // Avoid overflow. 1353 #ifdef AARCH64 1354 assert(DataLayout::counter_increment == 1, "required for cinc"); 1355 cinc(bumped_count, bumped_count, pl); 1356 #else 1357 add(bumped_count, bumped_count, DataLayout::counter_increment, pl); 1358 #endif // AARCH64 1359 } else { 1360 // Increment the register. Set condition codes. 1361 adds(bumped_count, bumped_count, DataLayout::counter_increment); 1362 // Avoid overflow. 1363 #ifdef AARCH64 1364 assert(DataLayout::counter_increment == 1, "required for cinv"); 1365 cinv(bumped_count, bumped_count, mi); // inverts 0x80..00 back to 0x7f..ff 1366 #else 1367 sub(bumped_count, bumped_count, DataLayout::counter_increment, mi); 1368 #endif // AARCH64 1369 } 1370 str(bumped_count, data); 1371 } 1372 1373 1374 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in, 1375 int offset, 1376 Register value, 1377 Register test_value_out, 1378 Label& not_equal_continue) { 1379 assert(ProfileInterpreter, "must be profiling interpreter"); 1380 assert_different_registers(mdp_in, test_value_out, value); 1381 1382 ldr(test_value_out, Address(mdp_in, offset)); 1383 cmp(test_value_out, value); 1384 1385 b(not_equal_continue, ne); 1386 } 1387 1388 1389 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) { 1390 assert(ProfileInterpreter, "must be profiling interpreter"); 1391 assert_different_registers(mdp_in, reg_temp); 1392 1393 ldr(reg_temp, Address(mdp_in, offset_of_disp)); 1394 add(mdp_in, mdp_in, reg_temp); 1395 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1396 } 1397 1398 1399 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) { 1400 assert(ProfileInterpreter, "must be profiling interpreter"); 1401 assert_different_registers(mdp_in, reg_offset, reg_tmp); 1402 1403 ldr(reg_tmp, Address(mdp_in, reg_offset)); 1404 add(mdp_in, mdp_in, reg_tmp); 1405 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1406 } 1407 1408 1409 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) { 1410 assert(ProfileInterpreter, "must be profiling interpreter"); 1411 add(mdp_in, mdp_in, constant); 1412 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1413 } 1414 1415 1416 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1417 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) { 1418 assert(ProfileInterpreter, "must be profiling interpreter"); 1419 assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp); 1420 1421 mov(R1, return_bci); 1422 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1); 1423 } 1424 1425 1426 // Sets mdp, bumped_count registers, blows Rtemp. 1427 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) { 1428 assert_different_registers(mdp, bumped_count); 1429 1430 if (ProfileInterpreter) { 1431 Label profile_continue; 1432 1433 // If no method data exists, go to profile_continue. 1434 // Otherwise, assign to mdp 1435 test_method_data_pointer(mdp, profile_continue); 1436 1437 // We are taking a branch. Increment the taken count. 1438 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count); 1439 1440 // The method data pointer needs to be updated to reflect the new target. 1441 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp); 1442 1443 bind (profile_continue); 1444 } 1445 } 1446 1447 1448 // Sets mdp, blows Rtemp. 1449 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) { 1450 assert_different_registers(mdp, Rtemp); 1451 1452 if (ProfileInterpreter) { 1453 Label profile_continue; 1454 1455 // If no method data exists, go to profile_continue. 1456 test_method_data_pointer(mdp, profile_continue); 1457 1458 // We are taking a branch. Increment the not taken count. 1459 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp); 1460 1461 // The method data pointer needs to be updated to correspond to the next bytecode 1462 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size())); 1463 1464 bind (profile_continue); 1465 } 1466 } 1467 1468 1469 // Sets mdp, blows Rtemp. 1470 void InterpreterMacroAssembler::profile_call(Register mdp) { 1471 assert_different_registers(mdp, Rtemp); 1472 1473 if (ProfileInterpreter) { 1474 Label profile_continue; 1475 1476 // If no method data exists, go to profile_continue. 1477 test_method_data_pointer(mdp, profile_continue); 1478 1479 // We are making a call. Increment the count. 1480 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1481 1482 // The method data pointer needs to be updated to reflect the new target. 1483 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size())); 1484 1485 bind (profile_continue); 1486 } 1487 } 1488 1489 1490 // Sets mdp, blows Rtemp. 1491 void InterpreterMacroAssembler::profile_final_call(Register mdp) { 1492 if (ProfileInterpreter) { 1493 Label profile_continue; 1494 1495 // If no method data exists, go to profile_continue. 1496 test_method_data_pointer(mdp, profile_continue); 1497 1498 // We are making a call. Increment the count. 1499 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1500 1501 // The method data pointer needs to be updated to reflect the new target. 1502 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1503 1504 bind (profile_continue); 1505 } 1506 } 1507 1508 1509 // Sets mdp, blows Rtemp. 1510 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) { 1511 assert_different_registers(mdp, receiver, Rtemp); 1512 1513 if (ProfileInterpreter) { 1514 Label profile_continue; 1515 1516 // If no method data exists, go to profile_continue. 1517 test_method_data_pointer(mdp, profile_continue); 1518 1519 Label skip_receiver_profile; 1520 if (receiver_can_be_null) { 1521 Label not_null; 1522 cbnz(receiver, not_null); 1523 // We are making a call. Increment the count for null receiver. 1524 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1525 b(skip_receiver_profile); 1526 bind(not_null); 1527 } 1528 1529 // Record the receiver type. 1530 record_klass_in_profile(receiver, mdp, Rtemp, true); 1531 bind(skip_receiver_profile); 1532 1533 // The method data pointer needs to be updated to reflect the new target. 1534 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1535 bind(profile_continue); 1536 } 1537 } 1538 1539 1540 void InterpreterMacroAssembler::record_klass_in_profile_helper( 1541 Register receiver, Register mdp, 1542 Register reg_tmp, 1543 int start_row, Label& done, bool is_virtual_call) { 1544 if (TypeProfileWidth == 0) 1545 return; 1546 1547 assert_different_registers(receiver, mdp, reg_tmp); 1548 1549 int last_row = VirtualCallData::row_limit() - 1; 1550 assert(start_row <= last_row, "must be work left to do"); 1551 // Test this row for both the receiver and for null. 1552 // Take any of three different outcomes: 1553 // 1. found receiver => increment count and goto done 1554 // 2. found null => keep looking for case 1, maybe allocate this cell 1555 // 3. found something else => keep looking for cases 1 and 2 1556 // Case 3 is handled by a recursive call. 1557 for (int row = start_row; row <= last_row; row++) { 1558 Label next_test; 1559 1560 // See if the receiver is receiver[n]. 1561 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row)); 1562 1563 test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test); 1564 1565 // The receiver is receiver[n]. Increment count[n]. 1566 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row)); 1567 increment_mdp_data_at(mdp, count_offset, reg_tmp); 1568 b(done); 1569 1570 bind(next_test); 1571 // reg_tmp now contains the receiver from the CallData. 1572 1573 if (row == start_row) { 1574 Label found_null; 1575 // Failed the equality check on receiver[n]... Test for null. 1576 if (start_row == last_row) { 1577 // The only thing left to do is handle the null case. 1578 if (is_virtual_call) { 1579 cbz(reg_tmp, found_null); 1580 // Receiver did not match any saved receiver and there is no empty row for it. 1581 // Increment total counter to indicate polymorphic case. 1582 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp); 1583 b(done); 1584 bind(found_null); 1585 } else { 1586 cbnz(reg_tmp, done); 1587 } 1588 break; 1589 } 1590 // Since null is rare, make it be the branch-taken case. 1591 cbz(reg_tmp, found_null); 1592 1593 // Put all the "Case 3" tests here. 1594 record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call); 1595 1596 // Found a null. Keep searching for a matching receiver, 1597 // but remember that this is an empty (unused) slot. 1598 bind(found_null); 1599 } 1600 } 1601 1602 // In the fall-through case, we found no matching receiver, but we 1603 // observed the receiver[start_row] is NULL. 1604 1605 // Fill in the receiver field and increment the count. 1606 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row)); 1607 set_mdp_data_at(mdp, recvr_offset, receiver); 1608 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row)); 1609 mov(reg_tmp, DataLayout::counter_increment); 1610 set_mdp_data_at(mdp, count_offset, reg_tmp); 1611 if (start_row > 0) { 1612 b(done); 1613 } 1614 } 1615 1616 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver, 1617 Register mdp, 1618 Register reg_tmp, 1619 bool is_virtual_call) { 1620 assert(ProfileInterpreter, "must be profiling"); 1621 assert_different_registers(receiver, mdp, reg_tmp); 1622 1623 Label done; 1624 1625 record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call); 1626 1627 bind (done); 1628 } 1629 1630 // Sets mdp, blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1631 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) { 1632 assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3); 1633 1634 if (ProfileInterpreter) { 1635 Label profile_continue; 1636 uint row; 1637 1638 // If no method data exists, go to profile_continue. 1639 test_method_data_pointer(mdp, profile_continue); 1640 1641 // Update the total ret count. 1642 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1643 1644 for (row = 0; row < RetData::row_limit(); row++) { 1645 Label next_test; 1646 1647 // See if return_bci is equal to bci[n]: 1648 test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci, 1649 Rtemp, next_test); 1650 1651 // return_bci is equal to bci[n]. Increment the count. 1652 increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp); 1653 1654 // The method data pointer needs to be updated to reflect the new target. 1655 update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp); 1656 b(profile_continue); 1657 bind(next_test); 1658 } 1659 1660 update_mdp_for_ret(return_bci); 1661 1662 bind(profile_continue); 1663 } 1664 } 1665 1666 1667 // Sets mdp. 1668 void InterpreterMacroAssembler::profile_null_seen(Register mdp) { 1669 if (ProfileInterpreter) { 1670 Label profile_continue; 1671 1672 // If no method data exists, go to profile_continue. 1673 test_method_data_pointer(mdp, profile_continue); 1674 1675 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant()); 1676 1677 // The method data pointer needs to be updated. 1678 int mdp_delta = in_bytes(BitData::bit_data_size()); 1679 if (TypeProfileCasts) { 1680 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1681 } 1682 update_mdp_by_constant(mdp, mdp_delta); 1683 1684 bind (profile_continue); 1685 } 1686 } 1687 1688 1689 // Sets mdp, blows Rtemp. 1690 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) { 1691 assert_different_registers(mdp, Rtemp); 1692 1693 if (ProfileInterpreter && TypeProfileCasts) { 1694 Label profile_continue; 1695 1696 // If no method data exists, go to profile_continue. 1697 test_method_data_pointer(mdp, profile_continue); 1698 1699 int count_offset = in_bytes(CounterData::count_offset()); 1700 // Back up the address, since we have already bumped the mdp. 1701 count_offset -= in_bytes(VirtualCallData::virtual_call_data_size()); 1702 1703 // *Decrement* the counter. We expect to see zero or small negatives. 1704 increment_mdp_data_at(mdp, count_offset, Rtemp, true); 1705 1706 bind (profile_continue); 1707 } 1708 } 1709 1710 1711 // Sets mdp, blows Rtemp. 1712 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) 1713 { 1714 assert_different_registers(mdp, klass, Rtemp); 1715 1716 if (ProfileInterpreter) { 1717 Label profile_continue; 1718 1719 // If no method data exists, go to profile_continue. 1720 test_method_data_pointer(mdp, profile_continue); 1721 1722 // The method data pointer needs to be updated. 1723 int mdp_delta = in_bytes(BitData::bit_data_size()); 1724 if (TypeProfileCasts) { 1725 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1726 1727 // Record the object type. 1728 record_klass_in_profile(klass, mdp, Rtemp, false); 1729 } 1730 update_mdp_by_constant(mdp, mdp_delta); 1731 1732 bind(profile_continue); 1733 } 1734 } 1735 1736 1737 // Sets mdp, blows Rtemp. 1738 void InterpreterMacroAssembler::profile_switch_default(Register mdp) { 1739 assert_different_registers(mdp, Rtemp); 1740 1741 if (ProfileInterpreter) { 1742 Label profile_continue; 1743 1744 // If no method data exists, go to profile_continue. 1745 test_method_data_pointer(mdp, profile_continue); 1746 1747 // Update the default case count 1748 increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp); 1749 1750 // The method data pointer needs to be updated. 1751 update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp); 1752 1753 bind(profile_continue); 1754 } 1755 } 1756 1757 1758 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2. 1759 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) { 1760 assert_different_registers(mdp, reg_tmp1, reg_tmp2); 1761 assert_different_registers(mdp, reg_tmp1, index); 1762 1763 if (ProfileInterpreter) { 1764 Label profile_continue; 1765 1766 const int count_offset = in_bytes(MultiBranchData::case_array_offset()) + 1767 in_bytes(MultiBranchData::relative_count_offset()); 1768 1769 const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) + 1770 in_bytes(MultiBranchData::relative_displacement_offset()); 1771 1772 // If no method data exists, go to profile_continue. 1773 test_method_data_pointer(mdp, profile_continue); 1774 1775 // Build the base (index * per_case_size_in_bytes()) 1776 logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size()))); 1777 1778 // Update the case count 1779 add(reg_tmp1, reg_tmp1, count_offset); 1780 increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2); 1781 1782 // The method data pointer needs to be updated. 1783 add(reg_tmp1, reg_tmp1, displacement_offset - count_offset); 1784 update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2); 1785 1786 bind (profile_continue); 1787 } 1788 } 1789 1790 1791 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) { 1792 #ifdef AARCH64 1793 rev_w(r, r); 1794 #else 1795 if (VM_Version::supports_rev()) { 1796 rev(r, r); 1797 } else { 1798 eor(rtmp1, r, AsmOperand(r, ror, 16)); 1799 mvn(rtmp2, 0x0000ff00); 1800 andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8)); 1801 eor(r, rtmp1, AsmOperand(r, ror, 8)); 1802 } 1803 #endif // AARCH64 1804 } 1805 1806 1807 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) { 1808 const intx addr = (intx) (address_of_counter + offset); 1809 1810 assert ((addr & 0x3) == 0, "address of counter should be aligned"); 1811 const intx offset_mask = right_n_bits(AARCH64_ONLY(12 + 2) NOT_AARCH64(12)); 1812 1813 const address base = (address) (addr & ~offset_mask); 1814 const int offs = (int) (addr & offset_mask); 1815 1816 const Register addr_base = tmp1; 1817 const Register val = tmp2; 1818 1819 mov_slow(addr_base, base); 1820 ldr_s32(val, Address(addr_base, offs)); 1821 1822 if (avoid_overflow) { 1823 adds_32(val, val, 1); 1824 #ifdef AARCH64 1825 Label L; 1826 b(L, mi); 1827 str_32(val, Address(addr_base, offs)); 1828 bind(L); 1829 #else 1830 str(val, Address(addr_base, offs), pl); 1831 #endif // AARCH64 1832 } else { 1833 add_32(val, val, 1); 1834 str_32(val, Address(addr_base, offs)); 1835 } 1836 } 1837 1838 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) { 1839 if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); } 1840 } 1841 1842 // Inline assembly for: 1843 // 1844 // if (thread is in interp_only_mode) { 1845 // InterpreterRuntime::post_method_entry(); 1846 // } 1847 // if (DTraceMethodProbes) { 1848 // SharedRuntime::dtrace_method_entry(method, receiver); 1849 // } 1850 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) { 1851 // SharedRuntime::rc_trace_method_entry(method, receiver); 1852 // } 1853 1854 void InterpreterMacroAssembler::notify_method_entry() { 1855 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1856 // track stack depth. If it is possible to enter interp_only_mode we add 1857 // the code to check if the event should be sent. 1858 if (can_post_interpreter_events()) { 1859 Label L; 1860 1861 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1862 cbz(Rtemp, L); 1863 1864 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry)); 1865 1866 bind(L); 1867 } 1868 1869 // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry 1870 if (DTraceMethodProbes) { 1871 Label Lcontinue; 1872 1873 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1874 cbz(Rtemp, Lcontinue); 1875 1876 mov(R0, Rthread); 1877 mov(R1, Rmethod); 1878 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1); 1879 1880 bind(Lcontinue); 1881 } 1882 // RedefineClasses() tracing support for obsolete method entry 1883 if (log_is_enabled(Trace, redefine, class, obsolete)) { 1884 mov(R0, Rthread); 1885 mov(R1, Rmethod); 1886 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), 1887 R0, R1); 1888 } 1889 } 1890 1891 1892 void InterpreterMacroAssembler::notify_method_exit( 1893 TosState state, NotifyMethodExitMode mode, 1894 bool native, Register result_lo, Register result_hi, FloatRegister result_fp) { 1895 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1896 // track stack depth. If it is possible to enter interp_only_mode we add 1897 // the code to check if the event should be sent. 1898 if (mode == NotifyJVMTI && can_post_interpreter_events()) { 1899 Label L; 1900 // Note: frame::interpreter_frame_result has a dependency on how the 1901 // method result is saved across the call to post_method_exit. If this 1902 // is changed then the interpreter_frame_result implementation will 1903 // need to be updated too. 1904 1905 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1906 cbz(Rtemp, L); 1907 1908 if (native) { 1909 // For c++ and template interpreter push both result registers on the 1910 // stack in native, we don't know the state. 1911 // On AArch64 result registers are stored into the frame at known locations. 1912 // See frame::interpreter_frame_result for code that gets the result values from here. 1913 assert(result_lo != noreg, "result registers should be defined"); 1914 1915 #ifdef AARCH64 1916 assert(result_hi == noreg, "result_hi is not used on AArch64"); 1917 assert(result_fp != fnoreg, "FP result register must be defined"); 1918 1919 str_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1920 str(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1921 #else 1922 assert(result_hi != noreg, "result registers should be defined"); 1923 1924 #ifdef __ABI_HARD__ 1925 assert(result_fp != fnoreg, "FP result register must be defined"); 1926 sub(SP, SP, 2 * wordSize); 1927 fstd(result_fp, Address(SP)); 1928 #endif // __ABI_HARD__ 1929 1930 push(RegisterSet(result_lo) | RegisterSet(result_hi)); 1931 #endif // AARCH64 1932 1933 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1934 1935 #ifdef AARCH64 1936 ldr_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1937 ldr(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1938 #else 1939 pop(RegisterSet(result_lo) | RegisterSet(result_hi)); 1940 #ifdef __ABI_HARD__ 1941 fldd(result_fp, Address(SP)); 1942 add(SP, SP, 2 * wordSize); 1943 #endif // __ABI_HARD__ 1944 #endif // AARCH64 1945 1946 } else { 1947 // For the template interpreter, the value on tos is the size of the 1948 // state. (c++ interpreter calls jvmti somewhere else). 1949 push(state); 1950 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1951 pop(state); 1952 } 1953 1954 bind(L); 1955 } 1956 1957 // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit 1958 if (DTraceMethodProbes) { 1959 Label Lcontinue; 1960 1961 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1962 cbz(Rtemp, Lcontinue); 1963 1964 push(state); 1965 1966 mov(R0, Rthread); 1967 mov(R1, Rmethod); 1968 1969 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1); 1970 1971 pop(state); 1972 1973 bind(Lcontinue); 1974 } 1975 } 1976 1977 1978 #ifndef PRODUCT 1979 1980 void InterpreterMacroAssembler::trace_state(const char* msg) { 1981 int push_size = save_caller_save_registers(); 1982 1983 Label Lcontinue; 1984 InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n"); 1985 InlinedString Lmsg(msg); 1986 InlinedAddress Lprintf((address)printf); 1987 1988 ldr_literal(R0, Lmsg0); 1989 ldr_literal(R1, Lmsg); 1990 mov(R2, FP); 1991 add(R3, SP, push_size); // original SP (without saved registers) 1992 ldr_literal(Rtemp, Lprintf); 1993 call(Rtemp); 1994 1995 b(Lcontinue); 1996 1997 bind_literal(Lmsg0); 1998 bind_literal(Lmsg); 1999 bind_literal(Lprintf); 2000 2001 2002 bind(Lcontinue); 2003 2004 restore_caller_save_registers(); 2005 } 2006 2007 #endif 2008 2009 // Jump if ((*counter_addr += increment) & mask) satisfies the condition. 2010 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr, 2011 int increment, Address mask_addr, 2012 Register scratch, Register scratch2, 2013 AsmCondition cond, Label* where) { 2014 // caution: scratch2 and base address of counter_addr can be the same 2015 assert_different_registers(scratch, scratch2); 2016 ldr_u32(scratch, counter_addr); 2017 add(scratch, scratch, increment); 2018 str_32(scratch, counter_addr); 2019 2020 #ifdef AARCH64 2021 ldr_u32(scratch2, mask_addr); 2022 ands_w(ZR, scratch, scratch2); 2023 #else 2024 ldr(scratch2, mask_addr); 2025 andrs(scratch, scratch, scratch2); 2026 #endif // AARCH64 2027 b(*where, cond); 2028 } 2029 2030 void InterpreterMacroAssembler::get_method_counters(Register method, 2031 Register Rcounters, 2032 Label& skip, 2033 bool saveRegs, 2034 Register reg1, 2035 Register reg2, 2036 Register reg3) { 2037 const Address method_counters(method, Method::method_counters_offset()); 2038 Label has_counters; 2039 2040 ldr(Rcounters, method_counters); 2041 cbnz(Rcounters, has_counters); 2042 2043 if (saveRegs) { 2044 // Save and restore in use caller-saved registers since they will be trashed by call_VM 2045 assert(reg1 != noreg, "must specify reg1"); 2046 assert(reg2 != noreg, "must specify reg2"); 2047 #ifdef AARCH64 2048 assert(reg3 != noreg, "must specify reg3"); 2049 stp(reg1, reg2, Address(Rstack_top, -2*wordSize, pre_indexed)); 2050 stp(reg3, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 2051 #else 2052 assert(reg3 == noreg, "must not specify reg3"); 2053 push(RegisterSet(reg1) | RegisterSet(reg2)); 2054 #endif 2055 } 2056 2057 mov(R1, method); 2058 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1); 2059 2060 if (saveRegs) { 2061 #ifdef AARCH64 2062 ldp(reg3, ZR, Address(Rstack_top, 2*wordSize, post_indexed)); 2063 ldp(reg1, reg2, Address(Rstack_top, 2*wordSize, post_indexed)); 2064 #else 2065 pop(RegisterSet(reg1) | RegisterSet(reg2)); 2066 #endif 2067 } 2068 2069 ldr(Rcounters, method_counters); 2070 cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory 2071 2072 bind(has_counters); 2073 }