1 /* 2 * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "jvm.h" 27 #include "gc/shared/barrierSet.inline.hpp" 28 #include "gc/shared/cardTableModRefBS.inline.hpp" 29 #include "gc/shared/collectedHeap.hpp" 30 #include "interp_masm_arm.hpp" 31 #include "interpreter/interpreter.hpp" 32 #include "interpreter/interpreterRuntime.hpp" 33 #include "logging/log.hpp" 34 #include "oops/arrayOop.hpp" 35 #include "oops/markOop.hpp" 36 #include "oops/method.hpp" 37 #include "oops/methodData.hpp" 38 #include "prims/jvmtiExport.hpp" 39 #include "prims/jvmtiThreadState.hpp" 40 #include "runtime/basicLock.hpp" 41 #include "runtime/biasedLocking.hpp" 42 #include "runtime/sharedRuntime.hpp" 43 44 #if INCLUDE_ALL_GCS 45 #include "gc/g1/g1CollectedHeap.inline.hpp" 46 #include "gc/g1/g1SATBCardTableModRefBS.hpp" 47 #include "gc/g1/heapRegion.hpp" 48 #endif // INCLUDE_ALL_GCS 49 50 //-------------------------------------------------------------------- 51 // Implementation of InterpreterMacroAssembler 52 53 54 55 56 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) { 57 } 58 59 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 60 #if defined(ASSERT) && !defined(AARCH64) 61 // Ensure that last_sp is not filled. 62 { Label L; 63 ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 64 cbz(Rtemp, L); 65 stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL"); 66 bind(L); 67 } 68 #endif // ASSERT && !AARCH64 69 70 // Rbcp must be saved/restored since it may change due to GC. 71 save_bcp(); 72 73 #ifdef AARCH64 74 check_no_cached_stack_top(Rtemp); 75 save_stack_top(); 76 check_extended_sp(Rtemp); 77 cut_sp_before_call(); 78 #endif // AARCH64 79 80 // super call 81 MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions); 82 83 #ifdef AARCH64 84 // Restore SP to extended SP 85 restore_sp_after_call(Rtemp); 86 check_stack_top(); 87 clear_cached_stack_top(); 88 #endif // AARCH64 89 90 // Restore interpreter specific registers. 91 restore_bcp(); 92 restore_method(); 93 } 94 95 void InterpreterMacroAssembler::jump_to_entry(address entry) { 96 assert(entry, "Entry must have been generated by now"); 97 b(entry); 98 } 99 100 void InterpreterMacroAssembler::check_and_handle_popframe() { 101 if (can_pop_frame()) { 102 Label L; 103 const Register popframe_cond = R2_tmp; 104 105 // Initiate popframe handling only if it is not already being processed. If the flag 106 // has the popframe_processing bit set, it means that this code is called *during* popframe 107 // handling - we don't want to reenter. 108 109 ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset())); 110 tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L); 111 tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L); 112 113 // Call Interpreter::remove_activation_preserving_args_entry() to get the 114 // address of the same-named entrypoint in the generated interpreter code. 115 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry)); 116 117 // Call indirectly to avoid generation ordering problem. 118 jump(R0); 119 120 bind(L); 121 } 122 } 123 124 125 // Blows R2, Rtemp. Sets TOS cached value. 126 void InterpreterMacroAssembler::load_earlyret_value(TosState state) { 127 const Register thread_state = R2_tmp; 128 129 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 130 131 const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset()); 132 const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset()); 133 const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset()); 134 #ifndef AARCH64 135 const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset() 136 + in_ByteSize(wordSize)); 137 #endif // !AARCH64 138 139 Register zero = zero_register(Rtemp); 140 141 switch (state) { 142 case atos: ldr(R0_tos, oop_addr); 143 str(zero, oop_addr); 144 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 145 break; 146 147 #ifdef AARCH64 148 case ltos: ldr(R0_tos, val_addr); break; 149 #else 150 case ltos: ldr(R1_tos_hi, val_addr_hi); // fall through 151 #endif // AARCH64 152 case btos: // fall through 153 case ztos: // fall through 154 case ctos: // fall through 155 case stos: // fall through 156 case itos: ldr_s32(R0_tos, val_addr); break; 157 #ifdef __SOFTFP__ 158 case dtos: ldr(R1_tos_hi, val_addr_hi); // fall through 159 case ftos: ldr(R0_tos, val_addr); break; 160 #else 161 case ftos: ldr_float (S0_tos, val_addr); break; 162 case dtos: ldr_double(D0_tos, val_addr); break; 163 #endif // __SOFTFP__ 164 case vtos: /* nothing to do */ break; 165 default : ShouldNotReachHere(); 166 } 167 // Clean up tos value in the thread object 168 str(zero, val_addr); 169 #ifndef AARCH64 170 str(zero, val_addr_hi); 171 #endif // !AARCH64 172 173 mov(Rtemp, (int) ilgl); 174 str_32(Rtemp, tos_addr); 175 } 176 177 178 // Blows R2, Rtemp. 179 void InterpreterMacroAssembler::check_and_handle_earlyret() { 180 if (can_force_early_return()) { 181 Label L; 182 const Register thread_state = R2_tmp; 183 184 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 185 cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit; 186 187 // Initiate earlyret handling only if it is not already being processed. 188 // If the flag has the earlyret_processing bit set, it means that this code 189 // is called *during* earlyret handling - we don't want to reenter. 190 191 ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset())); 192 cmp(Rtemp, JvmtiThreadState::earlyret_pending); 193 b(L, ne); 194 195 // Call Interpreter::remove_activation_early_entry() to get the address of the 196 // same-named entrypoint in the generated interpreter code. 197 198 ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset())); 199 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0); 200 201 jump(R0); 202 203 bind(L); 204 } 205 } 206 207 208 // Sets reg. Blows Rtemp. 209 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) { 210 assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode"); 211 assert(reg != Rtemp, "should be different registers"); 212 213 ldrb(Rtemp, Address(Rbcp, bcp_offset)); 214 ldrb(reg, Address(Rbcp, bcp_offset+1)); 215 orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte)); 216 } 217 218 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) { 219 assert_different_registers(index, tmp_reg); 220 if (index_size == sizeof(u2)) { 221 // load bytes of index separately to avoid unaligned access 222 ldrb(index, Address(Rbcp, bcp_offset+1)); 223 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 224 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 225 } else if (index_size == sizeof(u4)) { 226 // TODO-AARCH64: consider using unaligned access here 227 ldrb(index, Address(Rbcp, bcp_offset+3)); 228 ldrb(tmp_reg, Address(Rbcp, bcp_offset+2)); 229 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 230 ldrb(tmp_reg, Address(Rbcp, bcp_offset+1)); 231 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 232 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 233 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 234 // Check if the secondary index definition is still ~x, otherwise 235 // we have to change the following assembler code to calculate the 236 // plain index. 237 assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line"); 238 mvn_32(index, index); // convert to plain index 239 } else if (index_size == sizeof(u1)) { 240 ldrb(index, Address(Rbcp, bcp_offset)); 241 } else { 242 ShouldNotReachHere(); 243 } 244 } 245 246 // Sets cache, index. 247 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) { 248 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 249 assert_different_registers(cache, index); 250 251 get_index_at_bcp(index, bcp_offset, cache, index_size); 252 253 // load constant pool cache pointer 254 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 255 256 // convert from field index to ConstantPoolCacheEntry index 257 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 258 // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called 259 logical_shift_left(index, index, 2); 260 } 261 262 // Sets cache, index, bytecode. 263 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) { 264 get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size); 265 // caution index and bytecode can be the same 266 add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord)); 267 #ifdef AARCH64 268 add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())); 269 ldarb(bytecode, bytecode); 270 #else 271 ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()))); 272 TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true); 273 #endif // AARCH64 274 } 275 276 // Sets cache. Blows reg_tmp. 277 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) { 278 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 279 assert_different_registers(cache, reg_tmp); 280 281 get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size); 282 283 // load constant pool cache pointer 284 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 285 286 // skip past the header 287 add(cache, cache, in_bytes(ConstantPoolCache::base_offset())); 288 // convert from field index to ConstantPoolCacheEntry index 289 // and from word offset to byte offset 290 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 291 add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord)); 292 } 293 294 // Load object from cpool->resolved_references(index) 295 void InterpreterMacroAssembler::load_resolved_reference_at_index( 296 Register result, Register index) { 297 assert_different_registers(result, index); 298 get_constant_pool(result); 299 300 Register cache = result; 301 // load pointer for resolved_references[] objArray 302 ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes())); 303 ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes())); 304 resolve_oop_handle(cache); 305 // Add in the index 306 // convert from field index to resolved_references() index and from 307 // word index to byte offset. Since this is a java object, it can be compressed 308 add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop)); 309 load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT))); 310 } 311 312 void InterpreterMacroAssembler::load_resolved_klass_at_offset( 313 Register Rcpool, Register Rindex, Register Rklass) { 314 add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord)); 315 ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index 316 ldr(Rklass, Address(Rcpool, ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses 317 add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord)); 318 ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes())); 319 } 320 321 // Generate a subtype check: branch to not_subtype if sub_klass is 322 // not a subtype of super_klass. 323 // Profiling code for the subtype check failure (profile_typecheck_failed) 324 // should be explicitly generated by the caller in the not_subtype case. 325 // Blows Rtemp, tmp1, tmp2. 326 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, 327 Register Rsuper_klass, 328 Label ¬_subtype, 329 Register tmp1, 330 Register tmp2) { 331 332 assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp); 333 Label ok_is_subtype, loop, update_cache; 334 335 const Register super_check_offset = tmp1; 336 const Register cached_super = tmp2; 337 338 // Profile the not-null value's klass. 339 profile_typecheck(tmp1, Rsub_klass); 340 341 // Load the super-klass's check offset into 342 ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset())); 343 344 // Check for self 345 cmp(Rsub_klass, Rsuper_klass); 346 347 // Load from the sub-klass's super-class display list, or a 1-word cache of 348 // the secondary superclass list, or a failing value with a sentinel offset 349 // if the super-klass is an interface or exceptionally deep in the Java 350 // hierarchy and we have to scan the secondary superclass list the hard way. 351 // See if we get an immediate positive hit 352 ldr(cached_super, Address(Rsub_klass, super_check_offset)); 353 354 cond_cmp(Rsuper_klass, cached_super, ne); 355 b(ok_is_subtype, eq); 356 357 // Check for immediate negative hit 358 cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset())); 359 b(not_subtype, ne); 360 361 // Now do a linear scan of the secondary super-klass chain. 362 const Register supers_arr = tmp1; 363 const Register supers_cnt = tmp2; 364 const Register cur_super = Rtemp; 365 366 // Load objArrayOop of secondary supers. 367 ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset())); 368 369 ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length 370 #ifdef AARCH64 371 cbz(supers_cnt, not_subtype); 372 add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes()); 373 #else 374 cmp(supers_cnt, 0); 375 376 // Skip to the start of array elements and prefetch the first super-klass. 377 ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne); 378 b(not_subtype, eq); 379 #endif // AARCH64 380 381 bind(loop); 382 383 #ifdef AARCH64 384 ldr(cur_super, Address(supers_arr, wordSize, post_indexed)); 385 #endif // AARCH64 386 387 cmp(cur_super, Rsuper_klass); 388 b(update_cache, eq); 389 390 subs(supers_cnt, supers_cnt, 1); 391 392 #ifndef AARCH64 393 ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne); 394 #endif // !AARCH64 395 396 b(loop, ne); 397 398 b(not_subtype); 399 400 bind(update_cache); 401 // Must be equal but missed in cache. Update cache. 402 str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset())); 403 404 bind(ok_is_subtype); 405 } 406 407 408 // The 1st part of the store check. 409 // Sets card_table_base register. 410 void InterpreterMacroAssembler::store_check_part1(Register card_table_base) { 411 // Check barrier set type (should be card table) and element size 412 BarrierSet* bs = Universe::heap()->barrier_set(); 413 assert(bs->kind() == BarrierSet::CardTableForRS || 414 bs->kind() == BarrierSet::CardTableExtension, 415 "Wrong barrier set kind"); 416 417 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs); 418 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "Adjust store check code"); 419 420 // Load card table base address. 421 422 /* Performance note. 423 424 There is an alternative way of loading card table base address 425 from thread descriptor, which may look more efficient: 426 427 ldr(card_table_base, Address(Rthread, JavaThread::card_table_base_offset())); 428 429 However, performance measurements of micro benchmarks and specJVM98 430 showed that loading of card table base from thread descriptor is 431 7-18% slower compared to loading of literal embedded into the code. 432 Possible cause is a cache miss (card table base address resides in a 433 rarely accessed area of thread descriptor). 434 */ 435 // TODO-AARCH64 Investigate if mov_slow is faster than ldr from Rthread on AArch64 436 mov_address(card_table_base, (address)ct->byte_map_base, symbolic_Relocation::card_table_reference); 437 } 438 439 // The 2nd part of the store check. 440 void InterpreterMacroAssembler::store_check_part2(Register obj, Register card_table_base, Register tmp) { 441 assert_different_registers(obj, card_table_base, tmp); 442 443 assert(CardTableModRefBS::dirty_card_val() == 0, "Dirty card value must be 0 due to optimizations."); 444 #ifdef AARCH64 445 add(card_table_base, card_table_base, AsmOperand(obj, lsr, CardTableModRefBS::card_shift)); 446 Address card_table_addr(card_table_base); 447 #else 448 Address card_table_addr(card_table_base, obj, lsr, CardTableModRefBS::card_shift); 449 #endif 450 451 if (UseCondCardMark) { 452 if (UseConcMarkSweepGC) { 453 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), noreg); 454 } 455 Label already_dirty; 456 457 ldrb(tmp, card_table_addr); 458 cbz(tmp, already_dirty); 459 460 set_card(card_table_base, card_table_addr, tmp); 461 bind(already_dirty); 462 463 } else { 464 if (UseConcMarkSweepGC && CMSPrecleaningEnabled) { 465 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg); 466 } 467 set_card(card_table_base, card_table_addr, tmp); 468 } 469 } 470 471 void InterpreterMacroAssembler::set_card(Register card_table_base, Address card_table_addr, Register tmp) { 472 #ifdef AARCH64 473 strb(ZR, card_table_addr); 474 #else 475 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set()); 476 if ((((uintptr_t)ct->byte_map_base & 0xff) == 0)) { 477 // Card table is aligned so the lowest byte of the table address base is zero. 478 // This works only if the code is not saved for later use, possibly 479 // in a context where the base would no longer be aligned. 480 strb(card_table_base, card_table_addr); 481 } else { 482 mov(tmp, 0); 483 strb(tmp, card_table_addr); 484 } 485 #endif // AARCH64 486 } 487 488 ////////////////////////////////////////////////////////////////////////////////// 489 490 491 // Java Expression Stack 492 493 void InterpreterMacroAssembler::pop_ptr(Register r) { 494 assert(r != Rstack_top, "unpredictable instruction"); 495 ldr(r, Address(Rstack_top, wordSize, post_indexed)); 496 } 497 498 void InterpreterMacroAssembler::pop_i(Register r) { 499 assert(r != Rstack_top, "unpredictable instruction"); 500 ldr_s32(r, Address(Rstack_top, wordSize, post_indexed)); 501 zap_high_non_significant_bits(r); 502 } 503 504 #ifdef AARCH64 505 void InterpreterMacroAssembler::pop_l(Register r) { 506 assert(r != Rstack_top, "unpredictable instruction"); 507 ldr(r, Address(Rstack_top, 2*wordSize, post_indexed)); 508 } 509 #else 510 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) { 511 assert_different_registers(lo, hi); 512 assert(lo < hi, "lo must be < hi"); 513 pop(RegisterSet(lo) | RegisterSet(hi)); 514 } 515 #endif // AARCH64 516 517 void InterpreterMacroAssembler::pop_f(FloatRegister fd) { 518 #ifdef AARCH64 519 ldr_s(fd, Address(Rstack_top, wordSize, post_indexed)); 520 #else 521 fpops(fd); 522 #endif // AARCH64 523 } 524 525 void InterpreterMacroAssembler::pop_d(FloatRegister fd) { 526 #ifdef AARCH64 527 ldr_d(fd, Address(Rstack_top, 2*wordSize, post_indexed)); 528 #else 529 fpopd(fd); 530 #endif // AARCH64 531 } 532 533 534 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value. 535 void InterpreterMacroAssembler::pop(TosState state) { 536 switch (state) { 537 case atos: pop_ptr(R0_tos); break; 538 case btos: // fall through 539 case ztos: // fall through 540 case ctos: // fall through 541 case stos: // fall through 542 case itos: pop_i(R0_tos); break; 543 #ifdef AARCH64 544 case ltos: pop_l(R0_tos); break; 545 #else 546 case ltos: pop_l(R0_tos_lo, R1_tos_hi); break; 547 #endif // AARCH64 548 #ifdef __SOFTFP__ 549 case ftos: pop_i(R0_tos); break; 550 case dtos: pop_l(R0_tos_lo, R1_tos_hi); break; 551 #else 552 case ftos: pop_f(S0_tos); break; 553 case dtos: pop_d(D0_tos); break; 554 #endif // __SOFTFP__ 555 case vtos: /* nothing to do */ break; 556 default : ShouldNotReachHere(); 557 } 558 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 559 } 560 561 void InterpreterMacroAssembler::push_ptr(Register r) { 562 assert(r != Rstack_top, "unpredictable instruction"); 563 str(r, Address(Rstack_top, -wordSize, pre_indexed)); 564 check_stack_top_on_expansion(); 565 } 566 567 void InterpreterMacroAssembler::push_i(Register r) { 568 assert(r != Rstack_top, "unpredictable instruction"); 569 str_32(r, Address(Rstack_top, -wordSize, pre_indexed)); 570 check_stack_top_on_expansion(); 571 } 572 573 #ifdef AARCH64 574 void InterpreterMacroAssembler::push_l(Register r) { 575 assert(r != Rstack_top, "unpredictable instruction"); 576 stp(r, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 577 check_stack_top_on_expansion(); 578 } 579 #else 580 void InterpreterMacroAssembler::push_l(Register lo, Register hi) { 581 assert_different_registers(lo, hi); 582 assert(lo < hi, "lo must be < hi"); 583 push(RegisterSet(lo) | RegisterSet(hi)); 584 } 585 #endif // AARCH64 586 587 void InterpreterMacroAssembler::push_f() { 588 #ifdef AARCH64 589 str_s(S0_tos, Address(Rstack_top, -wordSize, pre_indexed)); 590 check_stack_top_on_expansion(); 591 #else 592 fpushs(S0_tos); 593 #endif // AARCH64 594 } 595 596 void InterpreterMacroAssembler::push_d() { 597 #ifdef AARCH64 598 str_d(D0_tos, Address(Rstack_top, -2*wordSize, pre_indexed)); 599 check_stack_top_on_expansion(); 600 #else 601 fpushd(D0_tos); 602 #endif // AARCH64 603 } 604 605 // Transition state -> vtos. Blows Rtemp. 606 void InterpreterMacroAssembler::push(TosState state) { 607 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 608 switch (state) { 609 case atos: push_ptr(R0_tos); break; 610 case btos: // fall through 611 case ztos: // fall through 612 case ctos: // fall through 613 case stos: // fall through 614 case itos: push_i(R0_tos); break; 615 #ifdef AARCH64 616 case ltos: push_l(R0_tos); break; 617 #else 618 case ltos: push_l(R0_tos_lo, R1_tos_hi); break; 619 #endif // AARCH64 620 #ifdef __SOFTFP__ 621 case ftos: push_i(R0_tos); break; 622 case dtos: push_l(R0_tos_lo, R1_tos_hi); break; 623 #else 624 case ftos: push_f(); break; 625 case dtos: push_d(); break; 626 #endif // __SOFTFP__ 627 case vtos: /* nothing to do */ break; 628 default : ShouldNotReachHere(); 629 } 630 } 631 632 633 #ifndef AARCH64 634 635 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value. 636 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) { 637 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 638 // According to interpreter calling conventions, result is returned in R0/R1, 639 // but templates expect ftos in S0, and dtos in D0. 640 if (state == ftos) { 641 fmsr(S0_tos, R0); 642 } else if (state == dtos) { 643 fmdrr(D0_tos, R0, R1); 644 } 645 #endif // !__SOFTFP__ && !__ABI_HARD__ 646 } 647 648 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions). 649 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) { 650 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 651 // According to interpreter calling conventions, result is returned in R0/R1, 652 // so ftos (S0) and dtos (D0) are moved to R0/R1. 653 if (state == ftos) { 654 fmrs(R0, S0_tos); 655 } else if (state == dtos) { 656 fmrrd(R0, R1, D0_tos); 657 } 658 #endif // !__SOFTFP__ && !__ABI_HARD__ 659 } 660 661 #endif // !AARCH64 662 663 664 // Helpers for swap and dup 665 void InterpreterMacroAssembler::load_ptr(int n, Register val) { 666 ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 667 } 668 669 void InterpreterMacroAssembler::store_ptr(int n, Register val) { 670 str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 671 } 672 673 674 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() { 675 #ifdef AARCH64 676 check_no_cached_stack_top(Rtemp); 677 save_stack_top(); 678 cut_sp_before_call(); 679 mov(Rparams, Rstack_top); 680 #endif // AARCH64 681 682 // set sender sp 683 mov(Rsender_sp, SP); 684 685 #ifndef AARCH64 686 // record last_sp 687 str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 688 #endif // !AARCH64 689 } 690 691 // Jump to from_interpreted entry of a call unless single stepping is possible 692 // in this thread in which case we must call the i2i entry 693 void InterpreterMacroAssembler::jump_from_interpreted(Register method) { 694 assert_different_registers(method, Rtemp); 695 696 prepare_to_jump_from_interpreted(); 697 698 if (can_post_interpreter_events()) { 699 // JVMTI events, such as single-stepping, are implemented partly by avoiding running 700 // compiled code in threads for which the event is enabled. Check here for 701 // interp_only_mode if these events CAN be enabled. 702 703 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 704 #ifdef AARCH64 705 { 706 Label not_interp_only_mode; 707 708 cbz(Rtemp, not_interp_only_mode); 709 indirect_jump(Address(method, Method::interpreter_entry_offset()), Rtemp); 710 711 bind(not_interp_only_mode); 712 } 713 #else 714 cmp(Rtemp, 0); 715 ldr(PC, Address(method, Method::interpreter_entry_offset()), ne); 716 #endif // AARCH64 717 } 718 719 indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp); 720 } 721 722 723 void InterpreterMacroAssembler::restore_dispatch() { 724 mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos)); 725 } 726 727 728 // The following two routines provide a hook so that an implementation 729 // can schedule the dispatch in two parts. 730 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) { 731 // Nothing ARM-specific to be done here. 732 } 733 734 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) { 735 dispatch_next(state, step); 736 } 737 738 void InterpreterMacroAssembler::dispatch_base(TosState state, 739 DispatchTableMode table_mode, 740 bool verifyoop) { 741 if (VerifyActivationFrameSize) { 742 Label L; 743 #ifdef AARCH64 744 mov(Rtemp, SP); 745 sub(Rtemp, FP, Rtemp); 746 #else 747 sub(Rtemp, FP, SP); 748 #endif // AARCH64 749 int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize; 750 cmp(Rtemp, min_frame_size); 751 b(L, ge); 752 stop("broken stack frame"); 753 bind(L); 754 } 755 756 if (verifyoop) { 757 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 758 } 759 760 if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) { 761 zap_high_non_significant_bits(R0_tos); 762 } 763 764 #ifdef ASSERT 765 Label L; 766 mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos)); 767 cmp(Rtemp, RdispatchTable); 768 b(L, eq); 769 stop("invalid RdispatchTable"); 770 bind(L); 771 #endif 772 773 if (table_mode == DispatchDefault) { 774 if (state == vtos) { 775 indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp); 776 } else { 777 #ifdef AARCH64 778 sub(Rtemp, R3_bytecode, (Interpreter::distance_from_dispatch_table(vtos) - 779 Interpreter::distance_from_dispatch_table(state))); 780 indirect_jump(Address::indexed_ptr(RdispatchTable, Rtemp), Rtemp); 781 #else 782 // on 32-bit ARM this method is faster than the one above. 783 sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) - 784 Interpreter::distance_from_dispatch_table(state)) * wordSize); 785 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 786 #endif 787 } 788 } else { 789 assert(table_mode == DispatchNormal, "invalid dispatch table mode"); 790 address table = (address) Interpreter::normal_table(state); 791 mov_slow(Rtemp, table); 792 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 793 } 794 795 nop(); // to avoid filling CPU pipeline with invalid instructions 796 nop(); 797 } 798 799 void InterpreterMacroAssembler::dispatch_only(TosState state) { 800 dispatch_base(state, DispatchDefault); 801 } 802 803 804 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) { 805 dispatch_base(state, DispatchNormal); 806 } 807 808 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) { 809 dispatch_base(state, DispatchNormal, false); 810 } 811 812 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) { 813 // load next bytecode and advance Rbcp 814 ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed)); 815 dispatch_base(state, DispatchDefault); 816 } 817 818 void InterpreterMacroAssembler::narrow(Register result) { 819 // mask integer result to narrower return type. 820 const Register Rtmp = R2; 821 822 // get method type 823 ldr(Rtmp, Address(Rmethod, Method::const_offset())); 824 ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset())); 825 826 Label notBool, notByte, notChar, done; 827 cmp(Rtmp, T_INT); 828 b(done, eq); 829 830 cmp(Rtmp, T_BOOLEAN); 831 b(notBool, ne); 832 and_32(result, result, 1); 833 b(done); 834 835 bind(notBool); 836 cmp(Rtmp, T_BYTE); 837 b(notByte, ne); 838 sign_extend(result, result, 8); 839 b(done); 840 841 bind(notByte); 842 cmp(Rtmp, T_CHAR); 843 b(notChar, ne); 844 zero_extend(result, result, 16); 845 b(done); 846 847 bind(notChar); 848 // cmp(Rtmp, T_SHORT); 849 // b(done, ne); 850 sign_extend(result, result, 16); 851 852 // Nothing to do 853 bind(done); 854 } 855 856 // remove activation 857 // 858 // Unlock the receiver if this is a synchronized method. 859 // Unlock any Java monitors from syncronized blocks. 860 // Remove the activation from the stack. 861 // 862 // If there are locked Java monitors 863 // If throw_monitor_exception 864 // throws IllegalMonitorStateException 865 // Else if install_monitor_exception 866 // installs IllegalMonitorStateException 867 // Else 868 // no error processing 869 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr, 870 bool throw_monitor_exception, 871 bool install_monitor_exception, 872 bool notify_jvmdi) { 873 Label unlock, unlocked, no_unlock; 874 875 // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result. 876 877 const Address do_not_unlock_if_synchronized(Rthread, 878 JavaThread::do_not_unlock_if_synchronized_offset()); 879 880 const Register Rflag = R2; 881 const Register Raccess_flags = R3; 882 883 restore_method(); 884 885 ldrb(Rflag, do_not_unlock_if_synchronized); 886 887 // get method access flags 888 ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset())); 889 890 strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag 891 892 // check if method is synchronized 893 894 tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked); 895 896 // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set. 897 cbnz(Rflag, no_unlock); 898 899 // unlock monitor 900 push(state); // save result 901 902 // BasicObjectLock will be first in list, since this is a synchronized method. However, need 903 // to check that the object has not been unlocked by an explicit monitorexit bytecode. 904 905 const Register Rmonitor = R1; // fixed in unlock_object() 906 const Register Robj = R2; 907 908 // address of first monitor 909 sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock)); 910 911 ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes())); 912 cbnz(Robj, unlock); 913 914 pop(state); 915 916 if (throw_monitor_exception) { 917 // Entry already unlocked, need to throw exception 918 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 919 should_not_reach_here(); 920 } else { 921 // Monitor already unlocked during a stack unroll. 922 // If requested, install an illegal_monitor_state_exception. 923 // Continue with stack unrolling. 924 if (install_monitor_exception) { 925 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 926 } 927 b(unlocked); 928 } 929 930 931 // Exception case for the check that all monitors are unlocked. 932 const Register Rcur = R2; 933 Label restart_check_monitors_unlocked, exception_monitor_is_still_locked; 934 935 bind(exception_monitor_is_still_locked); 936 // Monitor entry is still locked, need to throw exception. 937 // Rcur: monitor entry. 938 939 if (throw_monitor_exception) { 940 // Throw exception 941 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 942 should_not_reach_here(); 943 } else { 944 // Stack unrolling. Unlock object and install illegal_monitor_exception 945 // Unlock does not block, so don't have to worry about the frame 946 947 push(state); 948 mov(R1, Rcur); 949 unlock_object(R1); 950 951 if (install_monitor_exception) { 952 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 953 } 954 955 pop(state); 956 b(restart_check_monitors_unlocked); 957 } 958 959 bind(unlock); 960 unlock_object(Rmonitor); 961 pop(state); 962 963 // Check that for block-structured locking (i.e., that all locked objects has been unlocked) 964 bind(unlocked); 965 966 // Check that all monitors are unlocked 967 { 968 Label loop; 969 970 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; 971 const Register Rbottom = R3; 972 const Register Rcur_obj = Rtemp; 973 974 bind(restart_check_monitors_unlocked); 975 976 ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize)); 977 // points to current entry, starting with top-most entry 978 sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize); 979 // points to word before bottom of monitor block 980 981 cmp(Rcur, Rbottom); // check if there are no monitors 982 #ifndef AARCH64 983 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 984 // prefetch monitor's object 985 #endif // !AARCH64 986 b(no_unlock, eq); 987 988 bind(loop); 989 #ifdef AARCH64 990 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes())); 991 #endif // AARCH64 992 // check if current entry is used 993 cbnz(Rcur_obj, exception_monitor_is_still_locked); 994 995 add(Rcur, Rcur, entry_size); // otherwise advance to next entry 996 cmp(Rcur, Rbottom); // check if bottom reached 997 #ifndef AARCH64 998 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 999 // prefetch monitor's object 1000 #endif // !AARCH64 1001 b(loop, ne); // if not at bottom then check this entry 1002 } 1003 1004 bind(no_unlock); 1005 1006 // jvmti support 1007 if (notify_jvmdi) { 1008 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA 1009 } else { 1010 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA 1011 } 1012 1013 // remove activation 1014 #ifdef AARCH64 1015 ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize)); 1016 ldp(FP, LR, Address(FP)); 1017 mov(SP, Rtemp); 1018 #else 1019 mov(Rtemp, FP); 1020 ldmia(FP, RegisterSet(FP) | RegisterSet(LR)); 1021 ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize)); 1022 #endif 1023 1024 if (ret_addr != LR) { 1025 mov(ret_addr, LR); 1026 } 1027 } 1028 1029 1030 // At certain points in the method invocation the monitor of 1031 // synchronized methods hasn't been entered yet. 1032 // To correctly handle exceptions at these points, we set the thread local 1033 // variable _do_not_unlock_if_synchronized to true. The remove_activation will 1034 // check this flag. 1035 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) { 1036 const Address do_not_unlock_if_synchronized(Rthread, 1037 JavaThread::do_not_unlock_if_synchronized_offset()); 1038 if (flag) { 1039 mov(tmp, 1); 1040 strb(tmp, do_not_unlock_if_synchronized); 1041 } else { 1042 strb(zero_register(tmp), do_not_unlock_if_synchronized); 1043 } 1044 } 1045 1046 // Lock object 1047 // 1048 // Argument: R1 : Points to BasicObjectLock to be used for locking. 1049 // Must be initialized with object to lock. 1050 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1051 void InterpreterMacroAssembler::lock_object(Register Rlock) { 1052 assert(Rlock == R1, "the second argument"); 1053 1054 if (UseHeavyMonitors) { 1055 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1056 } else { 1057 Label done; 1058 1059 const Register Robj = R2; 1060 const Register Rmark = R3; 1061 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1062 1063 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1064 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1065 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1066 1067 Label already_locked, slow_case; 1068 1069 // Load object pointer 1070 ldr(Robj, Address(Rlock, obj_offset)); 1071 1072 if (UseBiasedLocking) { 1073 biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case); 1074 } 1075 1076 #ifdef AARCH64 1077 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1078 ldr(Rmark, Robj); 1079 1080 // Test if object is already locked 1081 assert(markOopDesc::unlocked_value == 1, "adjust this code"); 1082 tbz(Rmark, exact_log2(markOopDesc::unlocked_value), already_locked); 1083 1084 #else // AARCH64 1085 1086 // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread. 1087 // That would be acceptable as ether CAS or slow case path is taken in that case. 1088 // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as 1089 // loads are satisfied from a store queue if performed on the same processor). 1090 1091 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1092 ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes())); 1093 1094 // Test if object is already locked 1095 tst(Rmark, markOopDesc::unlocked_value); 1096 b(already_locked, eq); 1097 1098 #endif // !AARCH64 1099 // Save old object->mark() into BasicLock's displaced header 1100 str(Rmark, Address(Rlock, mark_offset)); 1101 1102 cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case); 1103 1104 #ifndef PRODUCT 1105 if (PrintBiasedLockingStatistics) { 1106 cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr()); 1107 } 1108 #endif //!PRODUCT 1109 1110 b(done); 1111 1112 // If we got here that means the object is locked by ether calling thread or another thread. 1113 bind(already_locked); 1114 // Handling of locked objects: recursive locks and slow case. 1115 1116 // Fast check for recursive lock. 1117 // 1118 // Can apply the optimization only if this is a stack lock 1119 // allocated in this thread. For efficiency, we can focus on 1120 // recently allocated stack locks (instead of reading the stack 1121 // base and checking whether 'mark' points inside the current 1122 // thread stack): 1123 // 1) (mark & 3) == 0 1124 // 2) SP <= mark < SP + os::pagesize() 1125 // 1126 // Warning: SP + os::pagesize can overflow the stack base. We must 1127 // neither apply the optimization for an inflated lock allocated 1128 // just above the thread stack (this is why condition 1 matters) 1129 // nor apply the optimization if the stack lock is inside the stack 1130 // of another thread. The latter is avoided even in case of overflow 1131 // because we have guard pages at the end of all stacks. Hence, if 1132 // we go over the stack base and hit the stack of another thread, 1133 // this should not be in a writeable area that could contain a 1134 // stack lock allocated by that thread. As a consequence, a stack 1135 // lock less than page size away from SP is guaranteed to be 1136 // owned by the current thread. 1137 // 1138 // Note: assuming SP is aligned, we can check the low bits of 1139 // (mark-SP) instead of the low bits of mark. In that case, 1140 // assuming page size is a power of 2, we can merge the two 1141 // conditions into a single test: 1142 // => ((mark - SP) & (3 - os::pagesize())) == 0 1143 1144 #ifdef AARCH64 1145 // Use the single check since the immediate is OK for AARCH64 1146 sub(R0, Rmark, Rstack_top); 1147 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size()); 1148 Assembler::LogicalImmediate imm(mask, false); 1149 ands(R0, R0, imm); 1150 1151 // For recursive case store 0 into lock record. 1152 // It is harmless to store it unconditionally as lock record contains some garbage 1153 // value in its _displaced_header field by this moment. 1154 str(ZR, Address(Rlock, mark_offset)); 1155 1156 #else // AARCH64 1157 // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand. 1158 // Check independently the low bits and the distance to SP. 1159 // -1- test low 2 bits 1160 movs(R0, AsmOperand(Rmark, lsl, 30)); 1161 // -2- test (mark - SP) if the low two bits are 0 1162 sub(R0, Rmark, SP, eq); 1163 movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq); 1164 // If still 'eq' then recursive locking OK: store 0 into lock record 1165 str(R0, Address(Rlock, mark_offset), eq); 1166 1167 #endif // AARCH64 1168 1169 #ifndef PRODUCT 1170 if (PrintBiasedLockingStatistics) { 1171 cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr()); 1172 } 1173 #endif // !PRODUCT 1174 1175 b(done, eq); 1176 1177 bind(slow_case); 1178 1179 // Call the runtime routine for slow case 1180 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1181 1182 bind(done); 1183 } 1184 } 1185 1186 1187 // Unlocks an object. Used in monitorexit bytecode and remove_activation. 1188 // 1189 // Argument: R1: Points to BasicObjectLock structure for lock 1190 // Throw an IllegalMonitorException if object is not locked by current thread 1191 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1192 void InterpreterMacroAssembler::unlock_object(Register Rlock) { 1193 assert(Rlock == R1, "the second argument"); 1194 1195 if (UseHeavyMonitors) { 1196 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1197 } else { 1198 Label done, slow_case; 1199 1200 const Register Robj = R2; 1201 const Register Rmark = R3; 1202 const Register Rresult = R0; 1203 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1204 1205 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1206 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1207 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1208 1209 const Register Rzero = zero_register(Rtemp); 1210 1211 // Load oop into Robj 1212 ldr(Robj, Address(Rlock, obj_offset)); 1213 1214 // Free entry 1215 str(Rzero, Address(Rlock, obj_offset)); 1216 1217 if (UseBiasedLocking) { 1218 biased_locking_exit(Robj, Rmark, done); 1219 } 1220 1221 // Load the old header from BasicLock structure 1222 ldr(Rmark, Address(Rlock, mark_offset)); 1223 1224 // Test for recursion (zero mark in BasicLock) 1225 cbz(Rmark, done); 1226 1227 bool allow_fallthrough_on_failure = true; 1228 1229 cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure); 1230 1231 b(done, eq); 1232 1233 bind(slow_case); 1234 1235 // Call the runtime routine for slow case. 1236 str(Robj, Address(Rlock, obj_offset)); // restore obj 1237 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1238 1239 bind(done); 1240 } 1241 } 1242 1243 1244 // Test ImethodDataPtr. If it is null, continue at the specified label 1245 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) { 1246 assert(ProfileInterpreter, "must be profiling interpreter"); 1247 ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1248 cbz(mdp, zero_continue); 1249 } 1250 1251 1252 // Set the method data pointer for the current bcp. 1253 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. 1254 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { 1255 assert(ProfileInterpreter, "must be profiling interpreter"); 1256 Label set_mdp; 1257 1258 // Test MDO to avoid the call if it is NULL. 1259 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1260 cbz(Rtemp, set_mdp); 1261 1262 mov(R0, Rmethod); 1263 mov(R1, Rbcp); 1264 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1); 1265 // R0/W0: mdi 1266 1267 // mdo is guaranteed to be non-zero here, we checked for it before the call. 1268 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1269 add(Rtemp, Rtemp, in_bytes(MethodData::data_offset())); 1270 add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0); 1271 1272 bind(set_mdp); 1273 str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1274 } 1275 1276 1277 void InterpreterMacroAssembler::verify_method_data_pointer() { 1278 assert(ProfileInterpreter, "must be profiling interpreter"); 1279 #ifdef ASSERT 1280 Label verify_continue; 1281 save_caller_save_registers(); 1282 1283 const Register Rmdp = R2; 1284 test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue 1285 1286 // If the mdp is valid, it will point to a DataLayout header which is 1287 // consistent with the bcp. The converse is highly probable also. 1288 1289 ldrh(R3, Address(Rmdp, DataLayout::bci_offset())); 1290 ldr(Rtemp, Address(Rmethod, Method::const_offset())); 1291 add(R3, R3, Rtemp); 1292 add(R3, R3, in_bytes(ConstMethod::codes_offset())); 1293 cmp(R3, Rbcp); 1294 b(verify_continue, eq); 1295 1296 mov(R0, Rmethod); 1297 mov(R1, Rbcp); 1298 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp); 1299 1300 bind(verify_continue); 1301 restore_caller_save_registers(); 1302 #endif // ASSERT 1303 } 1304 1305 1306 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) { 1307 assert(ProfileInterpreter, "must be profiling interpreter"); 1308 assert_different_registers(mdp_in, value); 1309 str(value, Address(mdp_in, offset)); 1310 } 1311 1312 1313 // Increments mdp data. Sets bumped_count register to adjusted counter. 1314 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in, 1315 int offset, 1316 Register bumped_count, 1317 bool decrement) { 1318 assert(ProfileInterpreter, "must be profiling interpreter"); 1319 1320 // Counter address 1321 Address data(mdp_in, offset); 1322 assert_different_registers(mdp_in, bumped_count); 1323 1324 increment_mdp_data_at(data, bumped_count, decrement); 1325 } 1326 1327 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) { 1328 assert_different_registers(mdp_in, Rtemp); 1329 assert(ProfileInterpreter, "must be profiling interpreter"); 1330 assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range"); 1331 1332 // Set the flag 1333 ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1334 orr(Rtemp, Rtemp, (unsigned)flag_byte_constant); 1335 strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1336 } 1337 1338 1339 // Increments mdp data. Sets bumped_count register to adjusted counter. 1340 void InterpreterMacroAssembler::increment_mdp_data_at(Address data, 1341 Register bumped_count, 1342 bool decrement) { 1343 assert(ProfileInterpreter, "must be profiling interpreter"); 1344 1345 ldr(bumped_count, data); 1346 if (decrement) { 1347 // Decrement the register. Set condition codes. 1348 subs(bumped_count, bumped_count, DataLayout::counter_increment); 1349 // Avoid overflow. 1350 #ifdef AARCH64 1351 assert(DataLayout::counter_increment == 1, "required for cinc"); 1352 cinc(bumped_count, bumped_count, pl); 1353 #else 1354 add(bumped_count, bumped_count, DataLayout::counter_increment, pl); 1355 #endif // AARCH64 1356 } else { 1357 // Increment the register. Set condition codes. 1358 adds(bumped_count, bumped_count, DataLayout::counter_increment); 1359 // Avoid overflow. 1360 #ifdef AARCH64 1361 assert(DataLayout::counter_increment == 1, "required for cinv"); 1362 cinv(bumped_count, bumped_count, mi); // inverts 0x80..00 back to 0x7f..ff 1363 #else 1364 sub(bumped_count, bumped_count, DataLayout::counter_increment, mi); 1365 #endif // AARCH64 1366 } 1367 str(bumped_count, data); 1368 } 1369 1370 1371 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in, 1372 int offset, 1373 Register value, 1374 Register test_value_out, 1375 Label& not_equal_continue) { 1376 assert(ProfileInterpreter, "must be profiling interpreter"); 1377 assert_different_registers(mdp_in, test_value_out, value); 1378 1379 ldr(test_value_out, Address(mdp_in, offset)); 1380 cmp(test_value_out, value); 1381 1382 b(not_equal_continue, ne); 1383 } 1384 1385 1386 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) { 1387 assert(ProfileInterpreter, "must be profiling interpreter"); 1388 assert_different_registers(mdp_in, reg_temp); 1389 1390 ldr(reg_temp, Address(mdp_in, offset_of_disp)); 1391 add(mdp_in, mdp_in, reg_temp); 1392 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1393 } 1394 1395 1396 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) { 1397 assert(ProfileInterpreter, "must be profiling interpreter"); 1398 assert_different_registers(mdp_in, reg_offset, reg_tmp); 1399 1400 ldr(reg_tmp, Address(mdp_in, reg_offset)); 1401 add(mdp_in, mdp_in, reg_tmp); 1402 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1403 } 1404 1405 1406 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) { 1407 assert(ProfileInterpreter, "must be profiling interpreter"); 1408 add(mdp_in, mdp_in, constant); 1409 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1410 } 1411 1412 1413 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1414 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) { 1415 assert(ProfileInterpreter, "must be profiling interpreter"); 1416 assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp); 1417 1418 mov(R1, return_bci); 1419 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1); 1420 } 1421 1422 1423 // Sets mdp, bumped_count registers, blows Rtemp. 1424 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) { 1425 assert_different_registers(mdp, bumped_count); 1426 1427 if (ProfileInterpreter) { 1428 Label profile_continue; 1429 1430 // If no method data exists, go to profile_continue. 1431 // Otherwise, assign to mdp 1432 test_method_data_pointer(mdp, profile_continue); 1433 1434 // We are taking a branch. Increment the taken count. 1435 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count); 1436 1437 // The method data pointer needs to be updated to reflect the new target. 1438 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp); 1439 1440 bind (profile_continue); 1441 } 1442 } 1443 1444 1445 // Sets mdp, blows Rtemp. 1446 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) { 1447 assert_different_registers(mdp, Rtemp); 1448 1449 if (ProfileInterpreter) { 1450 Label profile_continue; 1451 1452 // If no method data exists, go to profile_continue. 1453 test_method_data_pointer(mdp, profile_continue); 1454 1455 // We are taking a branch. Increment the not taken count. 1456 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp); 1457 1458 // The method data pointer needs to be updated to correspond to the next bytecode 1459 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size())); 1460 1461 bind (profile_continue); 1462 } 1463 } 1464 1465 1466 // Sets mdp, blows Rtemp. 1467 void InterpreterMacroAssembler::profile_call(Register mdp) { 1468 assert_different_registers(mdp, Rtemp); 1469 1470 if (ProfileInterpreter) { 1471 Label profile_continue; 1472 1473 // If no method data exists, go to profile_continue. 1474 test_method_data_pointer(mdp, profile_continue); 1475 1476 // We are making a call. Increment the count. 1477 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1478 1479 // The method data pointer needs to be updated to reflect the new target. 1480 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size())); 1481 1482 bind (profile_continue); 1483 } 1484 } 1485 1486 1487 // Sets mdp, blows Rtemp. 1488 void InterpreterMacroAssembler::profile_final_call(Register mdp) { 1489 if (ProfileInterpreter) { 1490 Label profile_continue; 1491 1492 // If no method data exists, go to profile_continue. 1493 test_method_data_pointer(mdp, profile_continue); 1494 1495 // We are making a call. Increment the count. 1496 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1497 1498 // The method data pointer needs to be updated to reflect the new target. 1499 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1500 1501 bind (profile_continue); 1502 } 1503 } 1504 1505 1506 // Sets mdp, blows Rtemp. 1507 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) { 1508 assert_different_registers(mdp, receiver, Rtemp); 1509 1510 if (ProfileInterpreter) { 1511 Label profile_continue; 1512 1513 // If no method data exists, go to profile_continue. 1514 test_method_data_pointer(mdp, profile_continue); 1515 1516 Label skip_receiver_profile; 1517 if (receiver_can_be_null) { 1518 Label not_null; 1519 cbnz(receiver, not_null); 1520 // We are making a call. Increment the count for null receiver. 1521 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1522 b(skip_receiver_profile); 1523 bind(not_null); 1524 } 1525 1526 // Record the receiver type. 1527 record_klass_in_profile(receiver, mdp, Rtemp, true); 1528 bind(skip_receiver_profile); 1529 1530 // The method data pointer needs to be updated to reflect the new target. 1531 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1532 bind(profile_continue); 1533 } 1534 } 1535 1536 1537 void InterpreterMacroAssembler::record_klass_in_profile_helper( 1538 Register receiver, Register mdp, 1539 Register reg_tmp, 1540 int start_row, Label& done, bool is_virtual_call) { 1541 if (TypeProfileWidth == 0) 1542 return; 1543 1544 assert_different_registers(receiver, mdp, reg_tmp); 1545 1546 int last_row = VirtualCallData::row_limit() - 1; 1547 assert(start_row <= last_row, "must be work left to do"); 1548 // Test this row for both the receiver and for null. 1549 // Take any of three different outcomes: 1550 // 1. found receiver => increment count and goto done 1551 // 2. found null => keep looking for case 1, maybe allocate this cell 1552 // 3. found something else => keep looking for cases 1 and 2 1553 // Case 3 is handled by a recursive call. 1554 for (int row = start_row; row <= last_row; row++) { 1555 Label next_test; 1556 1557 // See if the receiver is receiver[n]. 1558 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row)); 1559 1560 test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test); 1561 1562 // The receiver is receiver[n]. Increment count[n]. 1563 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row)); 1564 increment_mdp_data_at(mdp, count_offset, reg_tmp); 1565 b(done); 1566 1567 bind(next_test); 1568 // reg_tmp now contains the receiver from the CallData. 1569 1570 if (row == start_row) { 1571 Label found_null; 1572 // Failed the equality check on receiver[n]... Test for null. 1573 if (start_row == last_row) { 1574 // The only thing left to do is handle the null case. 1575 if (is_virtual_call) { 1576 cbz(reg_tmp, found_null); 1577 // Receiver did not match any saved receiver and there is no empty row for it. 1578 // Increment total counter to indicate polymorphic case. 1579 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp); 1580 b(done); 1581 bind(found_null); 1582 } else { 1583 cbnz(reg_tmp, done); 1584 } 1585 break; 1586 } 1587 // Since null is rare, make it be the branch-taken case. 1588 cbz(reg_tmp, found_null); 1589 1590 // Put all the "Case 3" tests here. 1591 record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call); 1592 1593 // Found a null. Keep searching for a matching receiver, 1594 // but remember that this is an empty (unused) slot. 1595 bind(found_null); 1596 } 1597 } 1598 1599 // In the fall-through case, we found no matching receiver, but we 1600 // observed the receiver[start_row] is NULL. 1601 1602 // Fill in the receiver field and increment the count. 1603 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row)); 1604 set_mdp_data_at(mdp, recvr_offset, receiver); 1605 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row)); 1606 mov(reg_tmp, DataLayout::counter_increment); 1607 set_mdp_data_at(mdp, count_offset, reg_tmp); 1608 if (start_row > 0) { 1609 b(done); 1610 } 1611 } 1612 1613 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver, 1614 Register mdp, 1615 Register reg_tmp, 1616 bool is_virtual_call) { 1617 assert(ProfileInterpreter, "must be profiling"); 1618 assert_different_registers(receiver, mdp, reg_tmp); 1619 1620 Label done; 1621 1622 record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call); 1623 1624 bind (done); 1625 } 1626 1627 // Sets mdp, blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1628 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) { 1629 assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3); 1630 1631 if (ProfileInterpreter) { 1632 Label profile_continue; 1633 uint row; 1634 1635 // If no method data exists, go to profile_continue. 1636 test_method_data_pointer(mdp, profile_continue); 1637 1638 // Update the total ret count. 1639 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1640 1641 for (row = 0; row < RetData::row_limit(); row++) { 1642 Label next_test; 1643 1644 // See if return_bci is equal to bci[n]: 1645 test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci, 1646 Rtemp, next_test); 1647 1648 // return_bci is equal to bci[n]. Increment the count. 1649 increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp); 1650 1651 // The method data pointer needs to be updated to reflect the new target. 1652 update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp); 1653 b(profile_continue); 1654 bind(next_test); 1655 } 1656 1657 update_mdp_for_ret(return_bci); 1658 1659 bind(profile_continue); 1660 } 1661 } 1662 1663 1664 // Sets mdp. 1665 void InterpreterMacroAssembler::profile_null_seen(Register mdp) { 1666 if (ProfileInterpreter) { 1667 Label profile_continue; 1668 1669 // If no method data exists, go to profile_continue. 1670 test_method_data_pointer(mdp, profile_continue); 1671 1672 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant()); 1673 1674 // The method data pointer needs to be updated. 1675 int mdp_delta = in_bytes(BitData::bit_data_size()); 1676 if (TypeProfileCasts) { 1677 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1678 } 1679 update_mdp_by_constant(mdp, mdp_delta); 1680 1681 bind (profile_continue); 1682 } 1683 } 1684 1685 1686 // Sets mdp, blows Rtemp. 1687 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) { 1688 assert_different_registers(mdp, Rtemp); 1689 1690 if (ProfileInterpreter && TypeProfileCasts) { 1691 Label profile_continue; 1692 1693 // If no method data exists, go to profile_continue. 1694 test_method_data_pointer(mdp, profile_continue); 1695 1696 int count_offset = in_bytes(CounterData::count_offset()); 1697 // Back up the address, since we have already bumped the mdp. 1698 count_offset -= in_bytes(VirtualCallData::virtual_call_data_size()); 1699 1700 // *Decrement* the counter. We expect to see zero or small negatives. 1701 increment_mdp_data_at(mdp, count_offset, Rtemp, true); 1702 1703 bind (profile_continue); 1704 } 1705 } 1706 1707 1708 // Sets mdp, blows Rtemp. 1709 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) 1710 { 1711 assert_different_registers(mdp, klass, Rtemp); 1712 1713 if (ProfileInterpreter) { 1714 Label profile_continue; 1715 1716 // If no method data exists, go to profile_continue. 1717 test_method_data_pointer(mdp, profile_continue); 1718 1719 // The method data pointer needs to be updated. 1720 int mdp_delta = in_bytes(BitData::bit_data_size()); 1721 if (TypeProfileCasts) { 1722 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1723 1724 // Record the object type. 1725 record_klass_in_profile(klass, mdp, Rtemp, false); 1726 } 1727 update_mdp_by_constant(mdp, mdp_delta); 1728 1729 bind(profile_continue); 1730 } 1731 } 1732 1733 1734 // Sets mdp, blows Rtemp. 1735 void InterpreterMacroAssembler::profile_switch_default(Register mdp) { 1736 assert_different_registers(mdp, Rtemp); 1737 1738 if (ProfileInterpreter) { 1739 Label profile_continue; 1740 1741 // If no method data exists, go to profile_continue. 1742 test_method_data_pointer(mdp, profile_continue); 1743 1744 // Update the default case count 1745 increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp); 1746 1747 // The method data pointer needs to be updated. 1748 update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp); 1749 1750 bind(profile_continue); 1751 } 1752 } 1753 1754 1755 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2. 1756 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) { 1757 assert_different_registers(mdp, reg_tmp1, reg_tmp2); 1758 assert_different_registers(mdp, reg_tmp1, index); 1759 1760 if (ProfileInterpreter) { 1761 Label profile_continue; 1762 1763 const int count_offset = in_bytes(MultiBranchData::case_array_offset()) + 1764 in_bytes(MultiBranchData::relative_count_offset()); 1765 1766 const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) + 1767 in_bytes(MultiBranchData::relative_displacement_offset()); 1768 1769 // If no method data exists, go to profile_continue. 1770 test_method_data_pointer(mdp, profile_continue); 1771 1772 // Build the base (index * per_case_size_in_bytes()) 1773 logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size()))); 1774 1775 // Update the case count 1776 add(reg_tmp1, reg_tmp1, count_offset); 1777 increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2); 1778 1779 // The method data pointer needs to be updated. 1780 add(reg_tmp1, reg_tmp1, displacement_offset - count_offset); 1781 update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2); 1782 1783 bind (profile_continue); 1784 } 1785 } 1786 1787 1788 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) { 1789 #ifdef AARCH64 1790 rev_w(r, r); 1791 #else 1792 if (VM_Version::supports_rev()) { 1793 rev(r, r); 1794 } else { 1795 eor(rtmp1, r, AsmOperand(r, ror, 16)); 1796 mvn(rtmp2, 0x0000ff00); 1797 andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8)); 1798 eor(r, rtmp1, AsmOperand(r, ror, 8)); 1799 } 1800 #endif // AARCH64 1801 } 1802 1803 1804 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) { 1805 const intx addr = (intx) (address_of_counter + offset); 1806 1807 assert ((addr & 0x3) == 0, "address of counter should be aligned"); 1808 const intx offset_mask = right_n_bits(AARCH64_ONLY(12 + 2) NOT_AARCH64(12)); 1809 1810 const address base = (address) (addr & ~offset_mask); 1811 const int offs = (int) (addr & offset_mask); 1812 1813 const Register addr_base = tmp1; 1814 const Register val = tmp2; 1815 1816 mov_slow(addr_base, base); 1817 ldr_s32(val, Address(addr_base, offs)); 1818 1819 if (avoid_overflow) { 1820 adds_32(val, val, 1); 1821 #ifdef AARCH64 1822 Label L; 1823 b(L, mi); 1824 str_32(val, Address(addr_base, offs)); 1825 bind(L); 1826 #else 1827 str(val, Address(addr_base, offs), pl); 1828 #endif // AARCH64 1829 } else { 1830 add_32(val, val, 1); 1831 str_32(val, Address(addr_base, offs)); 1832 } 1833 } 1834 1835 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) { 1836 if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); } 1837 } 1838 1839 // Inline assembly for: 1840 // 1841 // if (thread is in interp_only_mode) { 1842 // InterpreterRuntime::post_method_entry(); 1843 // } 1844 // if (DTraceMethodProbes) { 1845 // SharedRuntime::dtrace_method_entry(method, receiver); 1846 // } 1847 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) { 1848 // SharedRuntime::rc_trace_method_entry(method, receiver); 1849 // } 1850 1851 void InterpreterMacroAssembler::notify_method_entry() { 1852 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1853 // track stack depth. If it is possible to enter interp_only_mode we add 1854 // the code to check if the event should be sent. 1855 if (can_post_interpreter_events()) { 1856 Label L; 1857 1858 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1859 cbz(Rtemp, L); 1860 1861 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry)); 1862 1863 bind(L); 1864 } 1865 1866 // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry 1867 if (DTraceMethodProbes) { 1868 Label Lcontinue; 1869 1870 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1871 cbz(Rtemp, Lcontinue); 1872 1873 mov(R0, Rthread); 1874 mov(R1, Rmethod); 1875 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1); 1876 1877 bind(Lcontinue); 1878 } 1879 // RedefineClasses() tracing support for obsolete method entry 1880 if (log_is_enabled(Trace, redefine, class, obsolete)) { 1881 mov(R0, Rthread); 1882 mov(R1, Rmethod); 1883 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), 1884 R0, R1); 1885 } 1886 } 1887 1888 1889 void InterpreterMacroAssembler::notify_method_exit( 1890 TosState state, NotifyMethodExitMode mode, 1891 bool native, Register result_lo, Register result_hi, FloatRegister result_fp) { 1892 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1893 // track stack depth. If it is possible to enter interp_only_mode we add 1894 // the code to check if the event should be sent. 1895 if (mode == NotifyJVMTI && can_post_interpreter_events()) { 1896 Label L; 1897 // Note: frame::interpreter_frame_result has a dependency on how the 1898 // method result is saved across the call to post_method_exit. If this 1899 // is changed then the interpreter_frame_result implementation will 1900 // need to be updated too. 1901 1902 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1903 cbz(Rtemp, L); 1904 1905 if (native) { 1906 // For c++ and template interpreter push both result registers on the 1907 // stack in native, we don't know the state. 1908 // On AArch64 result registers are stored into the frame at known locations. 1909 // See frame::interpreter_frame_result for code that gets the result values from here. 1910 assert(result_lo != noreg, "result registers should be defined"); 1911 1912 #ifdef AARCH64 1913 assert(result_hi == noreg, "result_hi is not used on AArch64"); 1914 assert(result_fp != fnoreg, "FP result register must be defined"); 1915 1916 str_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1917 str(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1918 #else 1919 assert(result_hi != noreg, "result registers should be defined"); 1920 1921 #ifdef __ABI_HARD__ 1922 assert(result_fp != fnoreg, "FP result register must be defined"); 1923 sub(SP, SP, 2 * wordSize); 1924 fstd(result_fp, Address(SP)); 1925 #endif // __ABI_HARD__ 1926 1927 push(RegisterSet(result_lo) | RegisterSet(result_hi)); 1928 #endif // AARCH64 1929 1930 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1931 1932 #ifdef AARCH64 1933 ldr_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1934 ldr(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1935 #else 1936 pop(RegisterSet(result_lo) | RegisterSet(result_hi)); 1937 #ifdef __ABI_HARD__ 1938 fldd(result_fp, Address(SP)); 1939 add(SP, SP, 2 * wordSize); 1940 #endif // __ABI_HARD__ 1941 #endif // AARCH64 1942 1943 } else { 1944 // For the template interpreter, the value on tos is the size of the 1945 // state. (c++ interpreter calls jvmti somewhere else). 1946 push(state); 1947 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1948 pop(state); 1949 } 1950 1951 bind(L); 1952 } 1953 1954 // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit 1955 if (DTraceMethodProbes) { 1956 Label Lcontinue; 1957 1958 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1959 cbz(Rtemp, Lcontinue); 1960 1961 push(state); 1962 1963 mov(R0, Rthread); 1964 mov(R1, Rmethod); 1965 1966 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1); 1967 1968 pop(state); 1969 1970 bind(Lcontinue); 1971 } 1972 } 1973 1974 1975 #ifndef PRODUCT 1976 1977 void InterpreterMacroAssembler::trace_state(const char* msg) { 1978 int push_size = save_caller_save_registers(); 1979 1980 Label Lcontinue; 1981 InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n"); 1982 InlinedString Lmsg(msg); 1983 InlinedAddress Lprintf((address)printf); 1984 1985 ldr_literal(R0, Lmsg0); 1986 ldr_literal(R1, Lmsg); 1987 mov(R2, FP); 1988 add(R3, SP, push_size); // original SP (without saved registers) 1989 ldr_literal(Rtemp, Lprintf); 1990 call(Rtemp); 1991 1992 b(Lcontinue); 1993 1994 bind_literal(Lmsg0); 1995 bind_literal(Lmsg); 1996 bind_literal(Lprintf); 1997 1998 1999 bind(Lcontinue); 2000 2001 restore_caller_save_registers(); 2002 } 2003 2004 #endif 2005 2006 // Jump if ((*counter_addr += increment) & mask) satisfies the condition. 2007 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr, 2008 int increment, Address mask_addr, 2009 Register scratch, Register scratch2, 2010 AsmCondition cond, Label* where) { 2011 // caution: scratch2 and base address of counter_addr can be the same 2012 assert_different_registers(scratch, scratch2); 2013 ldr_u32(scratch, counter_addr); 2014 add(scratch, scratch, increment); 2015 str_32(scratch, counter_addr); 2016 2017 #ifdef AARCH64 2018 ldr_u32(scratch2, mask_addr); 2019 ands_w(ZR, scratch, scratch2); 2020 #else 2021 ldr(scratch2, mask_addr); 2022 andrs(scratch, scratch, scratch2); 2023 #endif // AARCH64 2024 b(*where, cond); 2025 } 2026 2027 void InterpreterMacroAssembler::get_method_counters(Register method, 2028 Register Rcounters, 2029 Label& skip, 2030 bool saveRegs, 2031 Register reg1, 2032 Register reg2, 2033 Register reg3) { 2034 const Address method_counters(method, Method::method_counters_offset()); 2035 Label has_counters; 2036 2037 ldr(Rcounters, method_counters); 2038 cbnz(Rcounters, has_counters); 2039 2040 if (saveRegs) { 2041 // Save and restore in use caller-saved registers since they will be trashed by call_VM 2042 assert(reg1 != noreg, "must specify reg1"); 2043 assert(reg2 != noreg, "must specify reg2"); 2044 #ifdef AARCH64 2045 assert(reg3 != noreg, "must specify reg3"); 2046 stp(reg1, reg2, Address(Rstack_top, -2*wordSize, pre_indexed)); 2047 stp(reg3, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 2048 #else 2049 assert(reg3 == noreg, "must not specify reg3"); 2050 push(RegisterSet(reg1) | RegisterSet(reg2)); 2051 #endif 2052 } 2053 2054 mov(R1, method); 2055 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1); 2056 2057 if (saveRegs) { 2058 #ifdef AARCH64 2059 ldp(reg3, ZR, Address(Rstack_top, 2*wordSize, post_indexed)); 2060 ldp(reg1, reg2, Address(Rstack_top, 2*wordSize, post_indexed)); 2061 #else 2062 pop(RegisterSet(reg1) | RegisterSet(reg2)); 2063 #endif 2064 } 2065 2066 ldr(Rcounters, method_counters); 2067 cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory 2068 2069 bind(has_counters); 2070 }