1 /* 2 * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "jvm.h" 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shared/cardTable.hpp" 29 #include "gc/shared/cardTableBarrierSet.inline.hpp" 30 #include "gc/shared/collectedHeap.hpp" 31 #include "interp_masm_arm.hpp" 32 #include "interpreter/interpreter.hpp" 33 #include "interpreter/interpreterRuntime.hpp" 34 #include "logging/log.hpp" 35 #include "oops/arrayOop.hpp" 36 #include "oops/markOop.hpp" 37 #include "oops/method.hpp" 38 #include "oops/methodData.hpp" 39 #include "prims/jvmtiExport.hpp" 40 #include "prims/jvmtiThreadState.hpp" 41 #include "runtime/basicLock.hpp" 42 #include "runtime/biasedLocking.hpp" 43 #include "runtime/frame.inline.hpp" 44 #include "runtime/sharedRuntime.hpp" 45 46 #if INCLUDE_ALL_GCS 47 #include "gc/g1/g1BarrierSet.hpp" 48 #include "gc/g1/heapRegion.hpp" 49 #endif // INCLUDE_ALL_GCS 50 51 //-------------------------------------------------------------------- 52 // Implementation of InterpreterMacroAssembler 53 54 55 56 57 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) { 58 } 59 60 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 61 #if defined(ASSERT) && !defined(AARCH64) 62 // Ensure that last_sp is not filled. 63 { Label L; 64 ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 65 cbz(Rtemp, L); 66 stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL"); 67 bind(L); 68 } 69 #endif // ASSERT && !AARCH64 70 71 // Rbcp must be saved/restored since it may change due to GC. 72 save_bcp(); 73 74 #ifdef AARCH64 75 check_no_cached_stack_top(Rtemp); 76 save_stack_top(); 77 check_extended_sp(Rtemp); 78 cut_sp_before_call(); 79 #endif // AARCH64 80 81 // super call 82 MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions); 83 84 #ifdef AARCH64 85 // Restore SP to extended SP 86 restore_sp_after_call(Rtemp); 87 check_stack_top(); 88 clear_cached_stack_top(); 89 #endif // AARCH64 90 91 // Restore interpreter specific registers. 92 restore_bcp(); 93 restore_method(); 94 } 95 96 void InterpreterMacroAssembler::jump_to_entry(address entry) { 97 assert(entry, "Entry must have been generated by now"); 98 b(entry); 99 } 100 101 void InterpreterMacroAssembler::check_and_handle_popframe() { 102 if (can_pop_frame()) { 103 Label L; 104 const Register popframe_cond = R2_tmp; 105 106 // Initiate popframe handling only if it is not already being processed. If the flag 107 // has the popframe_processing bit set, it means that this code is called *during* popframe 108 // handling - we don't want to reenter. 109 110 ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset())); 111 tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L); 112 tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L); 113 114 // Call Interpreter::remove_activation_preserving_args_entry() to get the 115 // address of the same-named entrypoint in the generated interpreter code. 116 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry)); 117 118 // Call indirectly to avoid generation ordering problem. 119 jump(R0); 120 121 bind(L); 122 } 123 } 124 125 126 // Blows R2, Rtemp. Sets TOS cached value. 127 void InterpreterMacroAssembler::load_earlyret_value(TosState state) { 128 const Register thread_state = R2_tmp; 129 130 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 131 132 const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset()); 133 const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset()); 134 const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset()); 135 #ifndef AARCH64 136 const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset() 137 + in_ByteSize(wordSize)); 138 #endif // !AARCH64 139 140 Register zero = zero_register(Rtemp); 141 142 switch (state) { 143 case atos: ldr(R0_tos, oop_addr); 144 str(zero, oop_addr); 145 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 146 break; 147 148 #ifdef AARCH64 149 case ltos: ldr(R0_tos, val_addr); break; 150 #else 151 case ltos: ldr(R1_tos_hi, val_addr_hi); // fall through 152 #endif // AARCH64 153 case btos: // fall through 154 case ztos: // fall through 155 case ctos: // fall through 156 case stos: // fall through 157 case itos: ldr_s32(R0_tos, val_addr); break; 158 #ifdef __SOFTFP__ 159 case dtos: ldr(R1_tos_hi, val_addr_hi); // fall through 160 case ftos: ldr(R0_tos, val_addr); break; 161 #else 162 case ftos: ldr_float (S0_tos, val_addr); break; 163 case dtos: ldr_double(D0_tos, val_addr); break; 164 #endif // __SOFTFP__ 165 case vtos: /* nothing to do */ break; 166 default : ShouldNotReachHere(); 167 } 168 // Clean up tos value in the thread object 169 str(zero, val_addr); 170 #ifndef AARCH64 171 str(zero, val_addr_hi); 172 #endif // !AARCH64 173 174 mov(Rtemp, (int) ilgl); 175 str_32(Rtemp, tos_addr); 176 } 177 178 179 // Blows R2, Rtemp. 180 void InterpreterMacroAssembler::check_and_handle_earlyret() { 181 if (can_force_early_return()) { 182 Label L; 183 const Register thread_state = R2_tmp; 184 185 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 186 cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit; 187 188 // Initiate earlyret handling only if it is not already being processed. 189 // If the flag has the earlyret_processing bit set, it means that this code 190 // is called *during* earlyret handling - we don't want to reenter. 191 192 ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset())); 193 cmp(Rtemp, JvmtiThreadState::earlyret_pending); 194 b(L, ne); 195 196 // Call Interpreter::remove_activation_early_entry() to get the address of the 197 // same-named entrypoint in the generated interpreter code. 198 199 ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset())); 200 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0); 201 202 jump(R0); 203 204 bind(L); 205 } 206 } 207 208 209 // Sets reg. Blows Rtemp. 210 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) { 211 assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode"); 212 assert(reg != Rtemp, "should be different registers"); 213 214 ldrb(Rtemp, Address(Rbcp, bcp_offset)); 215 ldrb(reg, Address(Rbcp, bcp_offset+1)); 216 orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte)); 217 } 218 219 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) { 220 assert_different_registers(index, tmp_reg); 221 if (index_size == sizeof(u2)) { 222 // load bytes of index separately to avoid unaligned access 223 ldrb(index, Address(Rbcp, bcp_offset+1)); 224 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 225 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 226 } else if (index_size == sizeof(u4)) { 227 // TODO-AARCH64: consider using unaligned access here 228 ldrb(index, Address(Rbcp, bcp_offset+3)); 229 ldrb(tmp_reg, Address(Rbcp, bcp_offset+2)); 230 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 231 ldrb(tmp_reg, Address(Rbcp, bcp_offset+1)); 232 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 233 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 234 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 235 // Check if the secondary index definition is still ~x, otherwise 236 // we have to change the following assembler code to calculate the 237 // plain index. 238 assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line"); 239 mvn_32(index, index); // convert to plain index 240 } else if (index_size == sizeof(u1)) { 241 ldrb(index, Address(Rbcp, bcp_offset)); 242 } else { 243 ShouldNotReachHere(); 244 } 245 } 246 247 // Sets cache, index. 248 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) { 249 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 250 assert_different_registers(cache, index); 251 252 get_index_at_bcp(index, bcp_offset, cache, index_size); 253 254 // load constant pool cache pointer 255 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 256 257 // convert from field index to ConstantPoolCacheEntry index 258 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 259 // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called 260 logical_shift_left(index, index, 2); 261 } 262 263 // Sets cache, index, bytecode. 264 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) { 265 get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size); 266 // caution index and bytecode can be the same 267 add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord)); 268 #ifdef AARCH64 269 add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())); 270 ldarb(bytecode, bytecode); 271 #else 272 ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()))); 273 TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true); 274 #endif // AARCH64 275 } 276 277 // Sets cache. Blows reg_tmp. 278 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) { 279 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 280 assert_different_registers(cache, reg_tmp); 281 282 get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size); 283 284 // load constant pool cache pointer 285 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 286 287 // skip past the header 288 add(cache, cache, in_bytes(ConstantPoolCache::base_offset())); 289 // convert from field index to ConstantPoolCacheEntry index 290 // and from word offset to byte offset 291 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 292 add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord)); 293 } 294 295 // Load object from cpool->resolved_references(index) 296 void InterpreterMacroAssembler::load_resolved_reference_at_index( 297 Register result, Register index) { 298 assert_different_registers(result, index); 299 get_constant_pool(result); 300 301 Register cache = result; 302 // load pointer for resolved_references[] objArray 303 ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes())); 304 ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes())); 305 resolve_oop_handle(cache); 306 // Add in the index 307 // convert from field index to resolved_references() index and from 308 // word index to byte offset. Since this is a java object, it can be compressed 309 add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop)); 310 load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT))); 311 } 312 313 void InterpreterMacroAssembler::load_resolved_klass_at_offset( 314 Register Rcpool, Register Rindex, Register Rklass) { 315 add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord)); 316 ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index 317 ldr(Rklass, Address(Rcpool, ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses 318 add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord)); 319 ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes())); 320 } 321 322 // Generate a subtype check: branch to not_subtype if sub_klass is 323 // not a subtype of super_klass. 324 // Profiling code for the subtype check failure (profile_typecheck_failed) 325 // should be explicitly generated by the caller in the not_subtype case. 326 // Blows Rtemp, tmp1, tmp2. 327 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, 328 Register Rsuper_klass, 329 Label ¬_subtype, 330 Register tmp1, 331 Register tmp2) { 332 333 assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp); 334 Label ok_is_subtype, loop, update_cache; 335 336 const Register super_check_offset = tmp1; 337 const Register cached_super = tmp2; 338 339 // Profile the not-null value's klass. 340 profile_typecheck(tmp1, Rsub_klass); 341 342 // Load the super-klass's check offset into 343 ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset())); 344 345 // Check for self 346 cmp(Rsub_klass, Rsuper_klass); 347 348 // Load from the sub-klass's super-class display list, or a 1-word cache of 349 // the secondary superclass list, or a failing value with a sentinel offset 350 // if the super-klass is an interface or exceptionally deep in the Java 351 // hierarchy and we have to scan the secondary superclass list the hard way. 352 // See if we get an immediate positive hit 353 ldr(cached_super, Address(Rsub_klass, super_check_offset)); 354 355 cond_cmp(Rsuper_klass, cached_super, ne); 356 b(ok_is_subtype, eq); 357 358 // Check for immediate negative hit 359 cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset())); 360 b(not_subtype, ne); 361 362 // Now do a linear scan of the secondary super-klass chain. 363 const Register supers_arr = tmp1; 364 const Register supers_cnt = tmp2; 365 const Register cur_super = Rtemp; 366 367 // Load objArrayOop of secondary supers. 368 ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset())); 369 370 ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length 371 #ifdef AARCH64 372 cbz(supers_cnt, not_subtype); 373 add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes()); 374 #else 375 cmp(supers_cnt, 0); 376 377 // Skip to the start of array elements and prefetch the first super-klass. 378 ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne); 379 b(not_subtype, eq); 380 #endif // AARCH64 381 382 bind(loop); 383 384 #ifdef AARCH64 385 ldr(cur_super, Address(supers_arr, wordSize, post_indexed)); 386 #endif // AARCH64 387 388 cmp(cur_super, Rsuper_klass); 389 b(update_cache, eq); 390 391 subs(supers_cnt, supers_cnt, 1); 392 393 #ifndef AARCH64 394 ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne); 395 #endif // !AARCH64 396 397 b(loop, ne); 398 399 b(not_subtype); 400 401 bind(update_cache); 402 // Must be equal but missed in cache. Update cache. 403 str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset())); 404 405 bind(ok_is_subtype); 406 } 407 408 409 ////////////////////////////////////////////////////////////////////////////////// 410 411 412 // Java Expression Stack 413 414 void InterpreterMacroAssembler::pop_ptr(Register r) { 415 assert(r != Rstack_top, "unpredictable instruction"); 416 ldr(r, Address(Rstack_top, wordSize, post_indexed)); 417 } 418 419 void InterpreterMacroAssembler::pop_i(Register r) { 420 assert(r != Rstack_top, "unpredictable instruction"); 421 ldr_s32(r, Address(Rstack_top, wordSize, post_indexed)); 422 zap_high_non_significant_bits(r); 423 } 424 425 #ifdef AARCH64 426 void InterpreterMacroAssembler::pop_l(Register r) { 427 assert(r != Rstack_top, "unpredictable instruction"); 428 ldr(r, Address(Rstack_top, 2*wordSize, post_indexed)); 429 } 430 #else 431 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) { 432 assert_different_registers(lo, hi); 433 assert(lo < hi, "lo must be < hi"); 434 pop(RegisterSet(lo) | RegisterSet(hi)); 435 } 436 #endif // AARCH64 437 438 void InterpreterMacroAssembler::pop_f(FloatRegister fd) { 439 #ifdef AARCH64 440 ldr_s(fd, Address(Rstack_top, wordSize, post_indexed)); 441 #else 442 fpops(fd); 443 #endif // AARCH64 444 } 445 446 void InterpreterMacroAssembler::pop_d(FloatRegister fd) { 447 #ifdef AARCH64 448 ldr_d(fd, Address(Rstack_top, 2*wordSize, post_indexed)); 449 #else 450 fpopd(fd); 451 #endif // AARCH64 452 } 453 454 455 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value. 456 void InterpreterMacroAssembler::pop(TosState state) { 457 switch (state) { 458 case atos: pop_ptr(R0_tos); break; 459 case btos: // fall through 460 case ztos: // fall through 461 case ctos: // fall through 462 case stos: // fall through 463 case itos: pop_i(R0_tos); break; 464 #ifdef AARCH64 465 case ltos: pop_l(R0_tos); break; 466 #else 467 case ltos: pop_l(R0_tos_lo, R1_tos_hi); break; 468 #endif // AARCH64 469 #ifdef __SOFTFP__ 470 case ftos: pop_i(R0_tos); break; 471 case dtos: pop_l(R0_tos_lo, R1_tos_hi); break; 472 #else 473 case ftos: pop_f(S0_tos); break; 474 case dtos: pop_d(D0_tos); break; 475 #endif // __SOFTFP__ 476 case vtos: /* nothing to do */ break; 477 default : ShouldNotReachHere(); 478 } 479 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 480 } 481 482 void InterpreterMacroAssembler::push_ptr(Register r) { 483 assert(r != Rstack_top, "unpredictable instruction"); 484 str(r, Address(Rstack_top, -wordSize, pre_indexed)); 485 check_stack_top_on_expansion(); 486 } 487 488 void InterpreterMacroAssembler::push_i(Register r) { 489 assert(r != Rstack_top, "unpredictable instruction"); 490 str_32(r, Address(Rstack_top, -wordSize, pre_indexed)); 491 check_stack_top_on_expansion(); 492 } 493 494 #ifdef AARCH64 495 void InterpreterMacroAssembler::push_l(Register r) { 496 assert(r != Rstack_top, "unpredictable instruction"); 497 stp(r, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 498 check_stack_top_on_expansion(); 499 } 500 #else 501 void InterpreterMacroAssembler::push_l(Register lo, Register hi) { 502 assert_different_registers(lo, hi); 503 assert(lo < hi, "lo must be < hi"); 504 push(RegisterSet(lo) | RegisterSet(hi)); 505 } 506 #endif // AARCH64 507 508 void InterpreterMacroAssembler::push_f() { 509 #ifdef AARCH64 510 str_s(S0_tos, Address(Rstack_top, -wordSize, pre_indexed)); 511 check_stack_top_on_expansion(); 512 #else 513 fpushs(S0_tos); 514 #endif // AARCH64 515 } 516 517 void InterpreterMacroAssembler::push_d() { 518 #ifdef AARCH64 519 str_d(D0_tos, Address(Rstack_top, -2*wordSize, pre_indexed)); 520 check_stack_top_on_expansion(); 521 #else 522 fpushd(D0_tos); 523 #endif // AARCH64 524 } 525 526 // Transition state -> vtos. Blows Rtemp. 527 void InterpreterMacroAssembler::push(TosState state) { 528 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 529 switch (state) { 530 case atos: push_ptr(R0_tos); break; 531 case btos: // fall through 532 case ztos: // fall through 533 case ctos: // fall through 534 case stos: // fall through 535 case itos: push_i(R0_tos); break; 536 #ifdef AARCH64 537 case ltos: push_l(R0_tos); break; 538 #else 539 case ltos: push_l(R0_tos_lo, R1_tos_hi); break; 540 #endif // AARCH64 541 #ifdef __SOFTFP__ 542 case ftos: push_i(R0_tos); break; 543 case dtos: push_l(R0_tos_lo, R1_tos_hi); break; 544 #else 545 case ftos: push_f(); break; 546 case dtos: push_d(); break; 547 #endif // __SOFTFP__ 548 case vtos: /* nothing to do */ break; 549 default : ShouldNotReachHere(); 550 } 551 } 552 553 554 #ifndef AARCH64 555 556 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value. 557 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) { 558 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 559 // According to interpreter calling conventions, result is returned in R0/R1, 560 // but templates expect ftos in S0, and dtos in D0. 561 if (state == ftos) { 562 fmsr(S0_tos, R0); 563 } else if (state == dtos) { 564 fmdrr(D0_tos, R0, R1); 565 } 566 #endif // !__SOFTFP__ && !__ABI_HARD__ 567 } 568 569 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions). 570 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) { 571 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 572 // According to interpreter calling conventions, result is returned in R0/R1, 573 // so ftos (S0) and dtos (D0) are moved to R0/R1. 574 if (state == ftos) { 575 fmrs(R0, S0_tos); 576 } else if (state == dtos) { 577 fmrrd(R0, R1, D0_tos); 578 } 579 #endif // !__SOFTFP__ && !__ABI_HARD__ 580 } 581 582 #endif // !AARCH64 583 584 585 // Helpers for swap and dup 586 void InterpreterMacroAssembler::load_ptr(int n, Register val) { 587 ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 588 } 589 590 void InterpreterMacroAssembler::store_ptr(int n, Register val) { 591 str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 592 } 593 594 595 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() { 596 #ifdef AARCH64 597 check_no_cached_stack_top(Rtemp); 598 save_stack_top(); 599 cut_sp_before_call(); 600 mov(Rparams, Rstack_top); 601 #endif // AARCH64 602 603 // set sender sp 604 mov(Rsender_sp, SP); 605 606 #ifndef AARCH64 607 // record last_sp 608 str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 609 #endif // !AARCH64 610 } 611 612 // Jump to from_interpreted entry of a call unless single stepping is possible 613 // in this thread in which case we must call the i2i entry 614 void InterpreterMacroAssembler::jump_from_interpreted(Register method) { 615 assert_different_registers(method, Rtemp); 616 617 prepare_to_jump_from_interpreted(); 618 619 if (can_post_interpreter_events()) { 620 // JVMTI events, such as single-stepping, are implemented partly by avoiding running 621 // compiled code in threads for which the event is enabled. Check here for 622 // interp_only_mode if these events CAN be enabled. 623 624 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 625 #ifdef AARCH64 626 { 627 Label not_interp_only_mode; 628 629 cbz(Rtemp, not_interp_only_mode); 630 indirect_jump(Address(method, Method::interpreter_entry_offset()), Rtemp); 631 632 bind(not_interp_only_mode); 633 } 634 #else 635 cmp(Rtemp, 0); 636 ldr(PC, Address(method, Method::interpreter_entry_offset()), ne); 637 #endif // AARCH64 638 } 639 640 indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp); 641 } 642 643 644 void InterpreterMacroAssembler::restore_dispatch() { 645 mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos)); 646 } 647 648 649 // The following two routines provide a hook so that an implementation 650 // can schedule the dispatch in two parts. 651 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) { 652 // Nothing ARM-specific to be done here. 653 } 654 655 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) { 656 dispatch_next(state, step); 657 } 658 659 void InterpreterMacroAssembler::dispatch_base(TosState state, 660 DispatchTableMode table_mode, 661 bool verifyoop) { 662 if (VerifyActivationFrameSize) { 663 Label L; 664 #ifdef AARCH64 665 mov(Rtemp, SP); 666 sub(Rtemp, FP, Rtemp); 667 #else 668 sub(Rtemp, FP, SP); 669 #endif // AARCH64 670 int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize; 671 cmp(Rtemp, min_frame_size); 672 b(L, ge); 673 stop("broken stack frame"); 674 bind(L); 675 } 676 677 if (verifyoop) { 678 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 679 } 680 681 if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) { 682 zap_high_non_significant_bits(R0_tos); 683 } 684 685 #ifdef ASSERT 686 Label L; 687 mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos)); 688 cmp(Rtemp, RdispatchTable); 689 b(L, eq); 690 stop("invalid RdispatchTable"); 691 bind(L); 692 #endif 693 694 if (table_mode == DispatchDefault) { 695 if (state == vtos) { 696 indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp); 697 } else { 698 #ifdef AARCH64 699 sub(Rtemp, R3_bytecode, (Interpreter::distance_from_dispatch_table(vtos) - 700 Interpreter::distance_from_dispatch_table(state))); 701 indirect_jump(Address::indexed_ptr(RdispatchTable, Rtemp), Rtemp); 702 #else 703 // on 32-bit ARM this method is faster than the one above. 704 sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) - 705 Interpreter::distance_from_dispatch_table(state)) * wordSize); 706 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 707 #endif 708 } 709 } else { 710 assert(table_mode == DispatchNormal, "invalid dispatch table mode"); 711 address table = (address) Interpreter::normal_table(state); 712 mov_slow(Rtemp, table); 713 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 714 } 715 716 nop(); // to avoid filling CPU pipeline with invalid instructions 717 nop(); 718 } 719 720 void InterpreterMacroAssembler::dispatch_only(TosState state) { 721 dispatch_base(state, DispatchDefault); 722 } 723 724 725 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) { 726 dispatch_base(state, DispatchNormal); 727 } 728 729 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) { 730 dispatch_base(state, DispatchNormal, false); 731 } 732 733 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) { 734 // load next bytecode and advance Rbcp 735 ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed)); 736 dispatch_base(state, DispatchDefault); 737 } 738 739 void InterpreterMacroAssembler::narrow(Register result) { 740 // mask integer result to narrower return type. 741 const Register Rtmp = R2; 742 743 // get method type 744 ldr(Rtmp, Address(Rmethod, Method::const_offset())); 745 ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset())); 746 747 Label notBool, notByte, notChar, done; 748 cmp(Rtmp, T_INT); 749 b(done, eq); 750 751 cmp(Rtmp, T_BOOLEAN); 752 b(notBool, ne); 753 and_32(result, result, 1); 754 b(done); 755 756 bind(notBool); 757 cmp(Rtmp, T_BYTE); 758 b(notByte, ne); 759 sign_extend(result, result, 8); 760 b(done); 761 762 bind(notByte); 763 cmp(Rtmp, T_CHAR); 764 b(notChar, ne); 765 zero_extend(result, result, 16); 766 b(done); 767 768 bind(notChar); 769 // cmp(Rtmp, T_SHORT); 770 // b(done, ne); 771 sign_extend(result, result, 16); 772 773 // Nothing to do 774 bind(done); 775 } 776 777 // remove activation 778 // 779 // Unlock the receiver if this is a synchronized method. 780 // Unlock any Java monitors from syncronized blocks. 781 // Remove the activation from the stack. 782 // 783 // If there are locked Java monitors 784 // If throw_monitor_exception 785 // throws IllegalMonitorStateException 786 // Else if install_monitor_exception 787 // installs IllegalMonitorStateException 788 // Else 789 // no error processing 790 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr, 791 bool throw_monitor_exception, 792 bool install_monitor_exception, 793 bool notify_jvmdi) { 794 Label unlock, unlocked, no_unlock; 795 796 // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result. 797 798 const Address do_not_unlock_if_synchronized(Rthread, 799 JavaThread::do_not_unlock_if_synchronized_offset()); 800 801 const Register Rflag = R2; 802 const Register Raccess_flags = R3; 803 804 restore_method(); 805 806 ldrb(Rflag, do_not_unlock_if_synchronized); 807 808 // get method access flags 809 ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset())); 810 811 strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag 812 813 // check if method is synchronized 814 815 tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked); 816 817 // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set. 818 cbnz(Rflag, no_unlock); 819 820 // unlock monitor 821 push(state); // save result 822 823 // BasicObjectLock will be first in list, since this is a synchronized method. However, need 824 // to check that the object has not been unlocked by an explicit monitorexit bytecode. 825 826 const Register Rmonitor = R1; // fixed in unlock_object() 827 const Register Robj = R2; 828 829 // address of first monitor 830 sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock)); 831 832 ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes())); 833 cbnz(Robj, unlock); 834 835 pop(state); 836 837 if (throw_monitor_exception) { 838 // Entry already unlocked, need to throw exception 839 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 840 should_not_reach_here(); 841 } else { 842 // Monitor already unlocked during a stack unroll. 843 // If requested, install an illegal_monitor_state_exception. 844 // Continue with stack unrolling. 845 if (install_monitor_exception) { 846 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 847 } 848 b(unlocked); 849 } 850 851 852 // Exception case for the check that all monitors are unlocked. 853 const Register Rcur = R2; 854 Label restart_check_monitors_unlocked, exception_monitor_is_still_locked; 855 856 bind(exception_monitor_is_still_locked); 857 // Monitor entry is still locked, need to throw exception. 858 // Rcur: monitor entry. 859 860 if (throw_monitor_exception) { 861 // Throw exception 862 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 863 should_not_reach_here(); 864 } else { 865 // Stack unrolling. Unlock object and install illegal_monitor_exception 866 // Unlock does not block, so don't have to worry about the frame 867 868 push(state); 869 mov(R1, Rcur); 870 unlock_object(R1); 871 872 if (install_monitor_exception) { 873 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 874 } 875 876 pop(state); 877 b(restart_check_monitors_unlocked); 878 } 879 880 bind(unlock); 881 unlock_object(Rmonitor); 882 pop(state); 883 884 // Check that for block-structured locking (i.e., that all locked objects has been unlocked) 885 bind(unlocked); 886 887 // Check that all monitors are unlocked 888 { 889 Label loop; 890 891 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; 892 const Register Rbottom = R3; 893 const Register Rcur_obj = Rtemp; 894 895 bind(restart_check_monitors_unlocked); 896 897 ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize)); 898 // points to current entry, starting with top-most entry 899 sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize); 900 // points to word before bottom of monitor block 901 902 cmp(Rcur, Rbottom); // check if there are no monitors 903 #ifndef AARCH64 904 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 905 // prefetch monitor's object 906 #endif // !AARCH64 907 b(no_unlock, eq); 908 909 bind(loop); 910 #ifdef AARCH64 911 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes())); 912 #endif // AARCH64 913 // check if current entry is used 914 cbnz(Rcur_obj, exception_monitor_is_still_locked); 915 916 add(Rcur, Rcur, entry_size); // otherwise advance to next entry 917 cmp(Rcur, Rbottom); // check if bottom reached 918 #ifndef AARCH64 919 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 920 // prefetch monitor's object 921 #endif // !AARCH64 922 b(loop, ne); // if not at bottom then check this entry 923 } 924 925 bind(no_unlock); 926 927 // jvmti support 928 if (notify_jvmdi) { 929 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA 930 } else { 931 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA 932 } 933 934 // remove activation 935 #ifdef AARCH64 936 ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize)); 937 ldp(FP, LR, Address(FP)); 938 mov(SP, Rtemp); 939 #else 940 mov(Rtemp, FP); 941 ldmia(FP, RegisterSet(FP) | RegisterSet(LR)); 942 ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize)); 943 #endif 944 945 if (ret_addr != LR) { 946 mov(ret_addr, LR); 947 } 948 } 949 950 951 // At certain points in the method invocation the monitor of 952 // synchronized methods hasn't been entered yet. 953 // To correctly handle exceptions at these points, we set the thread local 954 // variable _do_not_unlock_if_synchronized to true. The remove_activation will 955 // check this flag. 956 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) { 957 const Address do_not_unlock_if_synchronized(Rthread, 958 JavaThread::do_not_unlock_if_synchronized_offset()); 959 if (flag) { 960 mov(tmp, 1); 961 strb(tmp, do_not_unlock_if_synchronized); 962 } else { 963 strb(zero_register(tmp), do_not_unlock_if_synchronized); 964 } 965 } 966 967 // Lock object 968 // 969 // Argument: R1 : Points to BasicObjectLock to be used for locking. 970 // Must be initialized with object to lock. 971 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 972 void InterpreterMacroAssembler::lock_object(Register Rlock) { 973 assert(Rlock == R1, "the second argument"); 974 975 if (UseHeavyMonitors) { 976 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 977 } else { 978 Label done; 979 980 const Register Robj = R2; 981 const Register Rmark = R3; 982 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 983 984 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 985 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 986 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 987 988 Label already_locked, slow_case; 989 990 // Load object pointer 991 ldr(Robj, Address(Rlock, obj_offset)); 992 993 if (UseBiasedLocking) { 994 biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case); 995 } 996 997 #ifdef AARCH64 998 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 999 ldr(Rmark, Robj); 1000 1001 // Test if object is already locked 1002 assert(markOopDesc::unlocked_value == 1, "adjust this code"); 1003 tbz(Rmark, exact_log2(markOopDesc::unlocked_value), already_locked); 1004 1005 #else // AARCH64 1006 1007 // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread. 1008 // That would be acceptable as ether CAS or slow case path is taken in that case. 1009 // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as 1010 // loads are satisfied from a store queue if performed on the same processor). 1011 1012 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1013 ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes())); 1014 1015 // Test if object is already locked 1016 tst(Rmark, markOopDesc::unlocked_value); 1017 b(already_locked, eq); 1018 1019 #endif // !AARCH64 1020 // Save old object->mark() into BasicLock's displaced header 1021 str(Rmark, Address(Rlock, mark_offset)); 1022 1023 cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case); 1024 1025 #ifndef PRODUCT 1026 if (PrintBiasedLockingStatistics) { 1027 cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr()); 1028 } 1029 #endif //!PRODUCT 1030 1031 b(done); 1032 1033 // If we got here that means the object is locked by ether calling thread or another thread. 1034 bind(already_locked); 1035 // Handling of locked objects: recursive locks and slow case. 1036 1037 // Fast check for recursive lock. 1038 // 1039 // Can apply the optimization only if this is a stack lock 1040 // allocated in this thread. For efficiency, we can focus on 1041 // recently allocated stack locks (instead of reading the stack 1042 // base and checking whether 'mark' points inside the current 1043 // thread stack): 1044 // 1) (mark & 3) == 0 1045 // 2) SP <= mark < SP + os::pagesize() 1046 // 1047 // Warning: SP + os::pagesize can overflow the stack base. We must 1048 // neither apply the optimization for an inflated lock allocated 1049 // just above the thread stack (this is why condition 1 matters) 1050 // nor apply the optimization if the stack lock is inside the stack 1051 // of another thread. The latter is avoided even in case of overflow 1052 // because we have guard pages at the end of all stacks. Hence, if 1053 // we go over the stack base and hit the stack of another thread, 1054 // this should not be in a writeable area that could contain a 1055 // stack lock allocated by that thread. As a consequence, a stack 1056 // lock less than page size away from SP is guaranteed to be 1057 // owned by the current thread. 1058 // 1059 // Note: assuming SP is aligned, we can check the low bits of 1060 // (mark-SP) instead of the low bits of mark. In that case, 1061 // assuming page size is a power of 2, we can merge the two 1062 // conditions into a single test: 1063 // => ((mark - SP) & (3 - os::pagesize())) == 0 1064 1065 #ifdef AARCH64 1066 // Use the single check since the immediate is OK for AARCH64 1067 sub(R0, Rmark, Rstack_top); 1068 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size()); 1069 Assembler::LogicalImmediate imm(mask, false); 1070 ands(R0, R0, imm); 1071 1072 // For recursive case store 0 into lock record. 1073 // It is harmless to store it unconditionally as lock record contains some garbage 1074 // value in its _displaced_header field by this moment. 1075 str(ZR, Address(Rlock, mark_offset)); 1076 1077 #else // AARCH64 1078 // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand. 1079 // Check independently the low bits and the distance to SP. 1080 // -1- test low 2 bits 1081 movs(R0, AsmOperand(Rmark, lsl, 30)); 1082 // -2- test (mark - SP) if the low two bits are 0 1083 sub(R0, Rmark, SP, eq); 1084 movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq); 1085 // If still 'eq' then recursive locking OK: store 0 into lock record 1086 str(R0, Address(Rlock, mark_offset), eq); 1087 1088 #endif // AARCH64 1089 1090 #ifndef PRODUCT 1091 if (PrintBiasedLockingStatistics) { 1092 cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr()); 1093 } 1094 #endif // !PRODUCT 1095 1096 b(done, eq); 1097 1098 bind(slow_case); 1099 1100 // Call the runtime routine for slow case 1101 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1102 1103 bind(done); 1104 } 1105 } 1106 1107 1108 // Unlocks an object. Used in monitorexit bytecode and remove_activation. 1109 // 1110 // Argument: R1: Points to BasicObjectLock structure for lock 1111 // Throw an IllegalMonitorException if object is not locked by current thread 1112 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1113 void InterpreterMacroAssembler::unlock_object(Register Rlock) { 1114 assert(Rlock == R1, "the second argument"); 1115 1116 if (UseHeavyMonitors) { 1117 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1118 } else { 1119 Label done, slow_case; 1120 1121 const Register Robj = R2; 1122 const Register Rmark = R3; 1123 const Register Rresult = R0; 1124 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1125 1126 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1127 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1128 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1129 1130 const Register Rzero = zero_register(Rtemp); 1131 1132 // Load oop into Robj 1133 ldr(Robj, Address(Rlock, obj_offset)); 1134 1135 // Free entry 1136 str(Rzero, Address(Rlock, obj_offset)); 1137 1138 if (UseBiasedLocking) { 1139 biased_locking_exit(Robj, Rmark, done); 1140 } 1141 1142 // Load the old header from BasicLock structure 1143 ldr(Rmark, Address(Rlock, mark_offset)); 1144 1145 // Test for recursion (zero mark in BasicLock) 1146 cbz(Rmark, done); 1147 1148 bool allow_fallthrough_on_failure = true; 1149 1150 cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure); 1151 1152 b(done, eq); 1153 1154 bind(slow_case); 1155 1156 // Call the runtime routine for slow case. 1157 str(Robj, Address(Rlock, obj_offset)); // restore obj 1158 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1159 1160 bind(done); 1161 } 1162 } 1163 1164 1165 // Test ImethodDataPtr. If it is null, continue at the specified label 1166 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) { 1167 assert(ProfileInterpreter, "must be profiling interpreter"); 1168 ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1169 cbz(mdp, zero_continue); 1170 } 1171 1172 1173 // Set the method data pointer for the current bcp. 1174 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. 1175 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { 1176 assert(ProfileInterpreter, "must be profiling interpreter"); 1177 Label set_mdp; 1178 1179 // Test MDO to avoid the call if it is NULL. 1180 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1181 cbz(Rtemp, set_mdp); 1182 1183 mov(R0, Rmethod); 1184 mov(R1, Rbcp); 1185 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1); 1186 // R0/W0: mdi 1187 1188 // mdo is guaranteed to be non-zero here, we checked for it before the call. 1189 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1190 add(Rtemp, Rtemp, in_bytes(MethodData::data_offset())); 1191 add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0); 1192 1193 bind(set_mdp); 1194 str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1195 } 1196 1197 1198 void InterpreterMacroAssembler::verify_method_data_pointer() { 1199 assert(ProfileInterpreter, "must be profiling interpreter"); 1200 #ifdef ASSERT 1201 Label verify_continue; 1202 save_caller_save_registers(); 1203 1204 const Register Rmdp = R2; 1205 test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue 1206 1207 // If the mdp is valid, it will point to a DataLayout header which is 1208 // consistent with the bcp. The converse is highly probable also. 1209 1210 ldrh(R3, Address(Rmdp, DataLayout::bci_offset())); 1211 ldr(Rtemp, Address(Rmethod, Method::const_offset())); 1212 add(R3, R3, Rtemp); 1213 add(R3, R3, in_bytes(ConstMethod::codes_offset())); 1214 cmp(R3, Rbcp); 1215 b(verify_continue, eq); 1216 1217 mov(R0, Rmethod); 1218 mov(R1, Rbcp); 1219 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp); 1220 1221 bind(verify_continue); 1222 restore_caller_save_registers(); 1223 #endif // ASSERT 1224 } 1225 1226 1227 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) { 1228 assert(ProfileInterpreter, "must be profiling interpreter"); 1229 assert_different_registers(mdp_in, value); 1230 str(value, Address(mdp_in, offset)); 1231 } 1232 1233 1234 // Increments mdp data. Sets bumped_count register to adjusted counter. 1235 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in, 1236 int offset, 1237 Register bumped_count, 1238 bool decrement) { 1239 assert(ProfileInterpreter, "must be profiling interpreter"); 1240 1241 // Counter address 1242 Address data(mdp_in, offset); 1243 assert_different_registers(mdp_in, bumped_count); 1244 1245 increment_mdp_data_at(data, bumped_count, decrement); 1246 } 1247 1248 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) { 1249 assert_different_registers(mdp_in, Rtemp); 1250 assert(ProfileInterpreter, "must be profiling interpreter"); 1251 assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range"); 1252 1253 // Set the flag 1254 ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1255 orr(Rtemp, Rtemp, (unsigned)flag_byte_constant); 1256 strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1257 } 1258 1259 1260 // Increments mdp data. Sets bumped_count register to adjusted counter. 1261 void InterpreterMacroAssembler::increment_mdp_data_at(Address data, 1262 Register bumped_count, 1263 bool decrement) { 1264 assert(ProfileInterpreter, "must be profiling interpreter"); 1265 1266 ldr(bumped_count, data); 1267 if (decrement) { 1268 // Decrement the register. Set condition codes. 1269 subs(bumped_count, bumped_count, DataLayout::counter_increment); 1270 // Avoid overflow. 1271 #ifdef AARCH64 1272 assert(DataLayout::counter_increment == 1, "required for cinc"); 1273 cinc(bumped_count, bumped_count, pl); 1274 #else 1275 add(bumped_count, bumped_count, DataLayout::counter_increment, pl); 1276 #endif // AARCH64 1277 } else { 1278 // Increment the register. Set condition codes. 1279 adds(bumped_count, bumped_count, DataLayout::counter_increment); 1280 // Avoid overflow. 1281 #ifdef AARCH64 1282 assert(DataLayout::counter_increment == 1, "required for cinv"); 1283 cinv(bumped_count, bumped_count, mi); // inverts 0x80..00 back to 0x7f..ff 1284 #else 1285 sub(bumped_count, bumped_count, DataLayout::counter_increment, mi); 1286 #endif // AARCH64 1287 } 1288 str(bumped_count, data); 1289 } 1290 1291 1292 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in, 1293 int offset, 1294 Register value, 1295 Register test_value_out, 1296 Label& not_equal_continue) { 1297 assert(ProfileInterpreter, "must be profiling interpreter"); 1298 assert_different_registers(mdp_in, test_value_out, value); 1299 1300 ldr(test_value_out, Address(mdp_in, offset)); 1301 cmp(test_value_out, value); 1302 1303 b(not_equal_continue, ne); 1304 } 1305 1306 1307 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) { 1308 assert(ProfileInterpreter, "must be profiling interpreter"); 1309 assert_different_registers(mdp_in, reg_temp); 1310 1311 ldr(reg_temp, Address(mdp_in, offset_of_disp)); 1312 add(mdp_in, mdp_in, reg_temp); 1313 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1314 } 1315 1316 1317 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) { 1318 assert(ProfileInterpreter, "must be profiling interpreter"); 1319 assert_different_registers(mdp_in, reg_offset, reg_tmp); 1320 1321 ldr(reg_tmp, Address(mdp_in, reg_offset)); 1322 add(mdp_in, mdp_in, reg_tmp); 1323 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1324 } 1325 1326 1327 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) { 1328 assert(ProfileInterpreter, "must be profiling interpreter"); 1329 add(mdp_in, mdp_in, constant); 1330 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1331 } 1332 1333 1334 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1335 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) { 1336 assert(ProfileInterpreter, "must be profiling interpreter"); 1337 assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp); 1338 1339 mov(R1, return_bci); 1340 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1); 1341 } 1342 1343 1344 // Sets mdp, bumped_count registers, blows Rtemp. 1345 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) { 1346 assert_different_registers(mdp, bumped_count); 1347 1348 if (ProfileInterpreter) { 1349 Label profile_continue; 1350 1351 // If no method data exists, go to profile_continue. 1352 // Otherwise, assign to mdp 1353 test_method_data_pointer(mdp, profile_continue); 1354 1355 // We are taking a branch. Increment the taken count. 1356 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count); 1357 1358 // The method data pointer needs to be updated to reflect the new target. 1359 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp); 1360 1361 bind (profile_continue); 1362 } 1363 } 1364 1365 1366 // Sets mdp, blows Rtemp. 1367 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) { 1368 assert_different_registers(mdp, Rtemp); 1369 1370 if (ProfileInterpreter) { 1371 Label profile_continue; 1372 1373 // If no method data exists, go to profile_continue. 1374 test_method_data_pointer(mdp, profile_continue); 1375 1376 // We are taking a branch. Increment the not taken count. 1377 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp); 1378 1379 // The method data pointer needs to be updated to correspond to the next bytecode 1380 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size())); 1381 1382 bind (profile_continue); 1383 } 1384 } 1385 1386 1387 // Sets mdp, blows Rtemp. 1388 void InterpreterMacroAssembler::profile_call(Register mdp) { 1389 assert_different_registers(mdp, Rtemp); 1390 1391 if (ProfileInterpreter) { 1392 Label profile_continue; 1393 1394 // If no method data exists, go to profile_continue. 1395 test_method_data_pointer(mdp, profile_continue); 1396 1397 // We are making a call. Increment the count. 1398 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1399 1400 // The method data pointer needs to be updated to reflect the new target. 1401 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size())); 1402 1403 bind (profile_continue); 1404 } 1405 } 1406 1407 1408 // Sets mdp, blows Rtemp. 1409 void InterpreterMacroAssembler::profile_final_call(Register mdp) { 1410 if (ProfileInterpreter) { 1411 Label profile_continue; 1412 1413 // If no method data exists, go to profile_continue. 1414 test_method_data_pointer(mdp, profile_continue); 1415 1416 // We are making a call. Increment the count. 1417 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1418 1419 // The method data pointer needs to be updated to reflect the new target. 1420 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1421 1422 bind (profile_continue); 1423 } 1424 } 1425 1426 1427 // Sets mdp, blows Rtemp. 1428 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) { 1429 assert_different_registers(mdp, receiver, Rtemp); 1430 1431 if (ProfileInterpreter) { 1432 Label profile_continue; 1433 1434 // If no method data exists, go to profile_continue. 1435 test_method_data_pointer(mdp, profile_continue); 1436 1437 Label skip_receiver_profile; 1438 if (receiver_can_be_null) { 1439 Label not_null; 1440 cbnz(receiver, not_null); 1441 // We are making a call. Increment the count for null receiver. 1442 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1443 b(skip_receiver_profile); 1444 bind(not_null); 1445 } 1446 1447 // Record the receiver type. 1448 record_klass_in_profile(receiver, mdp, Rtemp, true); 1449 bind(skip_receiver_profile); 1450 1451 // The method data pointer needs to be updated to reflect the new target. 1452 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1453 bind(profile_continue); 1454 } 1455 } 1456 1457 1458 void InterpreterMacroAssembler::record_klass_in_profile_helper( 1459 Register receiver, Register mdp, 1460 Register reg_tmp, 1461 int start_row, Label& done, bool is_virtual_call) { 1462 if (TypeProfileWidth == 0) 1463 return; 1464 1465 assert_different_registers(receiver, mdp, reg_tmp); 1466 1467 int last_row = VirtualCallData::row_limit() - 1; 1468 assert(start_row <= last_row, "must be work left to do"); 1469 // Test this row for both the receiver and for null. 1470 // Take any of three different outcomes: 1471 // 1. found receiver => increment count and goto done 1472 // 2. found null => keep looking for case 1, maybe allocate this cell 1473 // 3. found something else => keep looking for cases 1 and 2 1474 // Case 3 is handled by a recursive call. 1475 for (int row = start_row; row <= last_row; row++) { 1476 Label next_test; 1477 1478 // See if the receiver is receiver[n]. 1479 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row)); 1480 1481 test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test); 1482 1483 // The receiver is receiver[n]. Increment count[n]. 1484 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row)); 1485 increment_mdp_data_at(mdp, count_offset, reg_tmp); 1486 b(done); 1487 1488 bind(next_test); 1489 // reg_tmp now contains the receiver from the CallData. 1490 1491 if (row == start_row) { 1492 Label found_null; 1493 // Failed the equality check on receiver[n]... Test for null. 1494 if (start_row == last_row) { 1495 // The only thing left to do is handle the null case. 1496 if (is_virtual_call) { 1497 cbz(reg_tmp, found_null); 1498 // Receiver did not match any saved receiver and there is no empty row for it. 1499 // Increment total counter to indicate polymorphic case. 1500 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp); 1501 b(done); 1502 bind(found_null); 1503 } else { 1504 cbnz(reg_tmp, done); 1505 } 1506 break; 1507 } 1508 // Since null is rare, make it be the branch-taken case. 1509 cbz(reg_tmp, found_null); 1510 1511 // Put all the "Case 3" tests here. 1512 record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call); 1513 1514 // Found a null. Keep searching for a matching receiver, 1515 // but remember that this is an empty (unused) slot. 1516 bind(found_null); 1517 } 1518 } 1519 1520 // In the fall-through case, we found no matching receiver, but we 1521 // observed the receiver[start_row] is NULL. 1522 1523 // Fill in the receiver field and increment the count. 1524 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row)); 1525 set_mdp_data_at(mdp, recvr_offset, receiver); 1526 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row)); 1527 mov(reg_tmp, DataLayout::counter_increment); 1528 set_mdp_data_at(mdp, count_offset, reg_tmp); 1529 if (start_row > 0) { 1530 b(done); 1531 } 1532 } 1533 1534 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver, 1535 Register mdp, 1536 Register reg_tmp, 1537 bool is_virtual_call) { 1538 assert(ProfileInterpreter, "must be profiling"); 1539 assert_different_registers(receiver, mdp, reg_tmp); 1540 1541 Label done; 1542 1543 record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call); 1544 1545 bind (done); 1546 } 1547 1548 // Sets mdp, blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1549 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) { 1550 assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3); 1551 1552 if (ProfileInterpreter) { 1553 Label profile_continue; 1554 uint row; 1555 1556 // If no method data exists, go to profile_continue. 1557 test_method_data_pointer(mdp, profile_continue); 1558 1559 // Update the total ret count. 1560 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1561 1562 for (row = 0; row < RetData::row_limit(); row++) { 1563 Label next_test; 1564 1565 // See if return_bci is equal to bci[n]: 1566 test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci, 1567 Rtemp, next_test); 1568 1569 // return_bci is equal to bci[n]. Increment the count. 1570 increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp); 1571 1572 // The method data pointer needs to be updated to reflect the new target. 1573 update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp); 1574 b(profile_continue); 1575 bind(next_test); 1576 } 1577 1578 update_mdp_for_ret(return_bci); 1579 1580 bind(profile_continue); 1581 } 1582 } 1583 1584 1585 // Sets mdp. 1586 void InterpreterMacroAssembler::profile_null_seen(Register mdp) { 1587 if (ProfileInterpreter) { 1588 Label profile_continue; 1589 1590 // If no method data exists, go to profile_continue. 1591 test_method_data_pointer(mdp, profile_continue); 1592 1593 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant()); 1594 1595 // The method data pointer needs to be updated. 1596 int mdp_delta = in_bytes(BitData::bit_data_size()); 1597 if (TypeProfileCasts) { 1598 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1599 } 1600 update_mdp_by_constant(mdp, mdp_delta); 1601 1602 bind (profile_continue); 1603 } 1604 } 1605 1606 1607 // Sets mdp, blows Rtemp. 1608 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) { 1609 assert_different_registers(mdp, Rtemp); 1610 1611 if (ProfileInterpreter && TypeProfileCasts) { 1612 Label profile_continue; 1613 1614 // If no method data exists, go to profile_continue. 1615 test_method_data_pointer(mdp, profile_continue); 1616 1617 int count_offset = in_bytes(CounterData::count_offset()); 1618 // Back up the address, since we have already bumped the mdp. 1619 count_offset -= in_bytes(VirtualCallData::virtual_call_data_size()); 1620 1621 // *Decrement* the counter. We expect to see zero or small negatives. 1622 increment_mdp_data_at(mdp, count_offset, Rtemp, true); 1623 1624 bind (profile_continue); 1625 } 1626 } 1627 1628 1629 // Sets mdp, blows Rtemp. 1630 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) 1631 { 1632 assert_different_registers(mdp, klass, Rtemp); 1633 1634 if (ProfileInterpreter) { 1635 Label profile_continue; 1636 1637 // If no method data exists, go to profile_continue. 1638 test_method_data_pointer(mdp, profile_continue); 1639 1640 // The method data pointer needs to be updated. 1641 int mdp_delta = in_bytes(BitData::bit_data_size()); 1642 if (TypeProfileCasts) { 1643 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1644 1645 // Record the object type. 1646 record_klass_in_profile(klass, mdp, Rtemp, false); 1647 } 1648 update_mdp_by_constant(mdp, mdp_delta); 1649 1650 bind(profile_continue); 1651 } 1652 } 1653 1654 1655 // Sets mdp, blows Rtemp. 1656 void InterpreterMacroAssembler::profile_switch_default(Register mdp) { 1657 assert_different_registers(mdp, Rtemp); 1658 1659 if (ProfileInterpreter) { 1660 Label profile_continue; 1661 1662 // If no method data exists, go to profile_continue. 1663 test_method_data_pointer(mdp, profile_continue); 1664 1665 // Update the default case count 1666 increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp); 1667 1668 // The method data pointer needs to be updated. 1669 update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp); 1670 1671 bind(profile_continue); 1672 } 1673 } 1674 1675 1676 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2. 1677 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) { 1678 assert_different_registers(mdp, reg_tmp1, reg_tmp2); 1679 assert_different_registers(mdp, reg_tmp1, index); 1680 1681 if (ProfileInterpreter) { 1682 Label profile_continue; 1683 1684 const int count_offset = in_bytes(MultiBranchData::case_array_offset()) + 1685 in_bytes(MultiBranchData::relative_count_offset()); 1686 1687 const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) + 1688 in_bytes(MultiBranchData::relative_displacement_offset()); 1689 1690 // If no method data exists, go to profile_continue. 1691 test_method_data_pointer(mdp, profile_continue); 1692 1693 // Build the base (index * per_case_size_in_bytes()) 1694 logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size()))); 1695 1696 // Update the case count 1697 add(reg_tmp1, reg_tmp1, count_offset); 1698 increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2); 1699 1700 // The method data pointer needs to be updated. 1701 add(reg_tmp1, reg_tmp1, displacement_offset - count_offset); 1702 update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2); 1703 1704 bind (profile_continue); 1705 } 1706 } 1707 1708 1709 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) { 1710 #ifdef AARCH64 1711 rev_w(r, r); 1712 #else 1713 if (VM_Version::supports_rev()) { 1714 rev(r, r); 1715 } else { 1716 eor(rtmp1, r, AsmOperand(r, ror, 16)); 1717 mvn(rtmp2, 0x0000ff00); 1718 andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8)); 1719 eor(r, rtmp1, AsmOperand(r, ror, 8)); 1720 } 1721 #endif // AARCH64 1722 } 1723 1724 1725 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) { 1726 const intx addr = (intx) (address_of_counter + offset); 1727 1728 assert ((addr & 0x3) == 0, "address of counter should be aligned"); 1729 const intx offset_mask = right_n_bits(AARCH64_ONLY(12 + 2) NOT_AARCH64(12)); 1730 1731 const address base = (address) (addr & ~offset_mask); 1732 const int offs = (int) (addr & offset_mask); 1733 1734 const Register addr_base = tmp1; 1735 const Register val = tmp2; 1736 1737 mov_slow(addr_base, base); 1738 ldr_s32(val, Address(addr_base, offs)); 1739 1740 if (avoid_overflow) { 1741 adds_32(val, val, 1); 1742 #ifdef AARCH64 1743 Label L; 1744 b(L, mi); 1745 str_32(val, Address(addr_base, offs)); 1746 bind(L); 1747 #else 1748 str(val, Address(addr_base, offs), pl); 1749 #endif // AARCH64 1750 } else { 1751 add_32(val, val, 1); 1752 str_32(val, Address(addr_base, offs)); 1753 } 1754 } 1755 1756 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) { 1757 if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); } 1758 } 1759 1760 // Inline assembly for: 1761 // 1762 // if (thread is in interp_only_mode) { 1763 // InterpreterRuntime::post_method_entry(); 1764 // } 1765 // if (DTraceMethodProbes) { 1766 // SharedRuntime::dtrace_method_entry(method, receiver); 1767 // } 1768 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) { 1769 // SharedRuntime::rc_trace_method_entry(method, receiver); 1770 // } 1771 1772 void InterpreterMacroAssembler::notify_method_entry() { 1773 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1774 // track stack depth. If it is possible to enter interp_only_mode we add 1775 // the code to check if the event should be sent. 1776 if (can_post_interpreter_events()) { 1777 Label L; 1778 1779 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1780 cbz(Rtemp, L); 1781 1782 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry)); 1783 1784 bind(L); 1785 } 1786 1787 // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry 1788 if (DTraceMethodProbes) { 1789 Label Lcontinue; 1790 1791 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1792 cbz(Rtemp, Lcontinue); 1793 1794 mov(R0, Rthread); 1795 mov(R1, Rmethod); 1796 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1); 1797 1798 bind(Lcontinue); 1799 } 1800 // RedefineClasses() tracing support for obsolete method entry 1801 if (log_is_enabled(Trace, redefine, class, obsolete)) { 1802 mov(R0, Rthread); 1803 mov(R1, Rmethod); 1804 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), 1805 R0, R1); 1806 } 1807 } 1808 1809 1810 void InterpreterMacroAssembler::notify_method_exit( 1811 TosState state, NotifyMethodExitMode mode, 1812 bool native, Register result_lo, Register result_hi, FloatRegister result_fp) { 1813 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1814 // track stack depth. If it is possible to enter interp_only_mode we add 1815 // the code to check if the event should be sent. 1816 if (mode == NotifyJVMTI && can_post_interpreter_events()) { 1817 Label L; 1818 // Note: frame::interpreter_frame_result has a dependency on how the 1819 // method result is saved across the call to post_method_exit. If this 1820 // is changed then the interpreter_frame_result implementation will 1821 // need to be updated too. 1822 1823 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1824 cbz(Rtemp, L); 1825 1826 if (native) { 1827 // For c++ and template interpreter push both result registers on the 1828 // stack in native, we don't know the state. 1829 // On AArch64 result registers are stored into the frame at known locations. 1830 // See frame::interpreter_frame_result for code that gets the result values from here. 1831 assert(result_lo != noreg, "result registers should be defined"); 1832 1833 #ifdef AARCH64 1834 assert(result_hi == noreg, "result_hi is not used on AArch64"); 1835 assert(result_fp != fnoreg, "FP result register must be defined"); 1836 1837 str_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1838 str(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1839 #else 1840 assert(result_hi != noreg, "result registers should be defined"); 1841 1842 #ifdef __ABI_HARD__ 1843 assert(result_fp != fnoreg, "FP result register must be defined"); 1844 sub(SP, SP, 2 * wordSize); 1845 fstd(result_fp, Address(SP)); 1846 #endif // __ABI_HARD__ 1847 1848 push(RegisterSet(result_lo) | RegisterSet(result_hi)); 1849 #endif // AARCH64 1850 1851 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1852 1853 #ifdef AARCH64 1854 ldr_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1855 ldr(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1856 #else 1857 pop(RegisterSet(result_lo) | RegisterSet(result_hi)); 1858 #ifdef __ABI_HARD__ 1859 fldd(result_fp, Address(SP)); 1860 add(SP, SP, 2 * wordSize); 1861 #endif // __ABI_HARD__ 1862 #endif // AARCH64 1863 1864 } else { 1865 // For the template interpreter, the value on tos is the size of the 1866 // state. (c++ interpreter calls jvmti somewhere else). 1867 push(state); 1868 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1869 pop(state); 1870 } 1871 1872 bind(L); 1873 } 1874 1875 // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit 1876 if (DTraceMethodProbes) { 1877 Label Lcontinue; 1878 1879 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1880 cbz(Rtemp, Lcontinue); 1881 1882 push(state); 1883 1884 mov(R0, Rthread); 1885 mov(R1, Rmethod); 1886 1887 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1); 1888 1889 pop(state); 1890 1891 bind(Lcontinue); 1892 } 1893 } 1894 1895 1896 #ifndef PRODUCT 1897 1898 void InterpreterMacroAssembler::trace_state(const char* msg) { 1899 int push_size = save_caller_save_registers(); 1900 1901 Label Lcontinue; 1902 InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n"); 1903 InlinedString Lmsg(msg); 1904 InlinedAddress Lprintf((address)printf); 1905 1906 ldr_literal(R0, Lmsg0); 1907 ldr_literal(R1, Lmsg); 1908 mov(R2, FP); 1909 add(R3, SP, push_size); // original SP (without saved registers) 1910 ldr_literal(Rtemp, Lprintf); 1911 call(Rtemp); 1912 1913 b(Lcontinue); 1914 1915 bind_literal(Lmsg0); 1916 bind_literal(Lmsg); 1917 bind_literal(Lprintf); 1918 1919 1920 bind(Lcontinue); 1921 1922 restore_caller_save_registers(); 1923 } 1924 1925 #endif 1926 1927 // Jump if ((*counter_addr += increment) & mask) satisfies the condition. 1928 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr, 1929 int increment, Address mask_addr, 1930 Register scratch, Register scratch2, 1931 AsmCondition cond, Label* where) { 1932 // caution: scratch2 and base address of counter_addr can be the same 1933 assert_different_registers(scratch, scratch2); 1934 ldr_u32(scratch, counter_addr); 1935 add(scratch, scratch, increment); 1936 str_32(scratch, counter_addr); 1937 1938 #ifdef AARCH64 1939 ldr_u32(scratch2, mask_addr); 1940 ands_w(ZR, scratch, scratch2); 1941 #else 1942 ldr(scratch2, mask_addr); 1943 andrs(scratch, scratch, scratch2); 1944 #endif // AARCH64 1945 b(*where, cond); 1946 } 1947 1948 void InterpreterMacroAssembler::get_method_counters(Register method, 1949 Register Rcounters, 1950 Label& skip, 1951 bool saveRegs, 1952 Register reg1, 1953 Register reg2, 1954 Register reg3) { 1955 const Address method_counters(method, Method::method_counters_offset()); 1956 Label has_counters; 1957 1958 ldr(Rcounters, method_counters); 1959 cbnz(Rcounters, has_counters); 1960 1961 if (saveRegs) { 1962 // Save and restore in use caller-saved registers since they will be trashed by call_VM 1963 assert(reg1 != noreg, "must specify reg1"); 1964 assert(reg2 != noreg, "must specify reg2"); 1965 #ifdef AARCH64 1966 assert(reg3 != noreg, "must specify reg3"); 1967 stp(reg1, reg2, Address(Rstack_top, -2*wordSize, pre_indexed)); 1968 stp(reg3, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 1969 #else 1970 assert(reg3 == noreg, "must not specify reg3"); 1971 push(RegisterSet(reg1) | RegisterSet(reg2)); 1972 #endif 1973 } 1974 1975 mov(R1, method); 1976 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1); 1977 1978 if (saveRegs) { 1979 #ifdef AARCH64 1980 ldp(reg3, ZR, Address(Rstack_top, 2*wordSize, post_indexed)); 1981 ldp(reg1, reg2, Address(Rstack_top, 2*wordSize, post_indexed)); 1982 #else 1983 pop(RegisterSet(reg1) | RegisterSet(reg2)); 1984 #endif 1985 } 1986 1987 ldr(Rcounters, method_counters); 1988 cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory 1989 1990 bind(has_counters); 1991 }