1 /* 2 * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "c1/c1_Compilation.hpp" 27 #include "c1/c1_LIRAssembler.hpp" 28 #include "c1/c1_MacroAssembler.hpp" 29 #include "c1/c1_Runtime1.hpp" 30 #include "c1/c1_ValueStack.hpp" 31 #include "ci/ciArrayKlass.hpp" 32 #include "ci/ciInstance.hpp" 33 #include "gc/shared/barrierSet.hpp" 34 #include "gc/shared/cardTableModRefBS.hpp" 35 #include "gc/shared/collectedHeap.hpp" 36 #include "nativeInst_arm.hpp" 37 #include "oops/objArrayKlass.hpp" 38 #include "runtime/sharedRuntime.hpp" 39 #include "vmreg_arm.inline.hpp" 40 41 #define __ _masm-> 42 43 // Note: Rtemp usage is this file should not impact C2 and should be 44 // correct as long as it is not implicitly used in lower layers (the 45 // arm [macro]assembler) and used with care in the other C1 specific 46 // files. 47 48 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { 49 ShouldNotCallThis(); // Not used on ARM 50 return false; 51 } 52 53 54 LIR_Opr LIR_Assembler::receiverOpr() { 55 // The first register in Java calling conventions 56 return FrameMap::R0_oop_opr; 57 } 58 59 LIR_Opr LIR_Assembler::osrBufferPointer() { 60 return FrameMap::as_pointer_opr(R0); 61 } 62 63 #ifndef PRODUCT 64 void LIR_Assembler::verify_reserved_argument_area_size(int args_count) { 65 assert(args_count * wordSize <= frame_map()->reserved_argument_area_size(), "not enough space for arguments"); 66 } 67 #endif // !PRODUCT 68 69 void LIR_Assembler::store_parameter(jint c, int offset_from_sp_in_words) { 70 assert(offset_from_sp_in_words >= 0, "invalid offset from sp"); 71 int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord; 72 assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space"); 73 __ mov_slow(Rtemp, c); 74 __ str(Rtemp, Address(SP, offset_from_sp_in_bytes)); 75 } 76 77 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_sp_in_words) { 78 assert(offset_from_sp_in_words >= 0, "invalid offset from sp"); 79 int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord; 80 assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space"); 81 __ mov_metadata(Rtemp, m); 82 __ str(Rtemp, Address(SP, offset_from_sp_in_bytes)); 83 } 84 85 //--------------fpu register translations----------------------- 86 87 88 void LIR_Assembler::set_24bit_FPU() { 89 ShouldNotReachHere(); 90 } 91 92 void LIR_Assembler::reset_FPU() { 93 ShouldNotReachHere(); 94 } 95 96 void LIR_Assembler::fpop() { 97 Unimplemented(); 98 } 99 100 void LIR_Assembler::fxch(int i) { 101 Unimplemented(); 102 } 103 104 void LIR_Assembler::fld(int i) { 105 Unimplemented(); 106 } 107 108 void LIR_Assembler::ffree(int i) { 109 Unimplemented(); 110 } 111 112 void LIR_Assembler::breakpoint() { 113 __ breakpoint(); 114 } 115 116 void LIR_Assembler::push(LIR_Opr opr) { 117 Unimplemented(); 118 } 119 120 void LIR_Assembler::pop(LIR_Opr opr) { 121 Unimplemented(); 122 } 123 124 //------------------------------------------- 125 Address LIR_Assembler::as_Address(LIR_Address* addr) { 126 Register base = addr->base()->as_pointer_register(); 127 128 #ifdef AARCH64 129 int align = exact_log2(type2aelembytes(addr->type(), true)); 130 #endif 131 132 if (addr->index()->is_illegal() || addr->index()->is_constant()) { 133 int offset = addr->disp(); 134 if (addr->index()->is_constant()) { 135 offset += addr->index()->as_constant_ptr()->as_jint() << addr->scale(); 136 } 137 138 #ifdef AARCH64 139 if (!Assembler::is_unsigned_imm_in_range(offset, 12, align) && !Assembler::is_imm_in_range(offset, 9, 0)) { 140 BAILOUT_("offset not in range", Address(base)); 141 } 142 assert(UseUnalignedAccesses || (offset & right_n_bits(align)) == 0, "offset should be aligned"); 143 #else 144 if ((offset <= -4096) || (offset >= 4096)) { 145 BAILOUT_("offset not in range", Address(base)); 146 } 147 #endif // AARCH64 148 149 return Address(base, offset); 150 151 } else { 152 assert(addr->disp() == 0, "can't have both"); 153 int scale = addr->scale(); 154 155 #ifdef AARCH64 156 assert((scale == 0) || (scale == align), "scale should be zero or equal to embedded shift"); 157 158 bool is_index_extended = (addr->index()->type() == T_INT); 159 if (is_index_extended) { 160 assert(addr->index()->is_single_cpu(), "should be"); 161 return Address(base, addr->index()->as_register(), ex_sxtw, scale); 162 } else { 163 assert(addr->index()->is_double_cpu(), "should be"); 164 return Address(base, addr->index()->as_register_lo(), ex_lsl, scale); 165 } 166 #else 167 assert(addr->index()->is_single_cpu(), "should be"); 168 return scale >= 0 ? Address(base, addr->index()->as_register(), lsl, scale) : 169 Address(base, addr->index()->as_register(), lsr, -scale); 170 #endif // AARCH64 171 } 172 } 173 174 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 175 #ifdef AARCH64 176 ShouldNotCallThis(); // Not used on AArch64 177 return Address(); 178 #else 179 Address base = as_Address(addr); 180 assert(base.index() == noreg, "must be"); 181 if (base.disp() + BytesPerWord >= 4096) { BAILOUT_("offset not in range", Address(base.base(),0)); } 182 return Address(base.base(), base.disp() + BytesPerWord); 183 #endif // AARCH64 184 } 185 186 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 187 #ifdef AARCH64 188 ShouldNotCallThis(); // Not used on AArch64 189 return Address(); 190 #else 191 return as_Address(addr); 192 #endif // AARCH64 193 } 194 195 196 void LIR_Assembler::osr_entry() { 197 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 198 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 199 ValueStack* entry_state = osr_entry->end()->state(); 200 int number_of_locks = entry_state->locks_size(); 201 202 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 203 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 204 205 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 206 int monitor_offset = (method()->max_locals() + 2 * (number_of_locks - 1)) * BytesPerWord; 207 for (int i = 0; i < number_of_locks; i++) { 208 int slot_offset = monitor_offset - (i * 2 * BytesPerWord); 209 __ ldr(R1, Address(OSR_buf, slot_offset + 0*BytesPerWord)); 210 __ ldr(R2, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 211 __ str(R1, frame_map()->address_for_monitor_lock(i)); 212 __ str(R2, frame_map()->address_for_monitor_object(i)); 213 } 214 } 215 216 217 int LIR_Assembler::check_icache() { 218 Register receiver = LIR_Assembler::receiverOpr()->as_register(); 219 int offset = __ offset(); 220 __ inline_cache_check(receiver, Ricklass); 221 return offset; 222 } 223 224 225 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) { 226 jobject o = (jobject)Universe::non_oop_word(); 227 int index = __ oop_recorder()->allocate_oop_index(o); 228 229 PatchingStub* patch = new PatchingStub(_masm, patching_id(info), index); 230 231 __ patchable_mov_oop(reg, o, index); 232 patching_epilog(patch, lir_patch_normal, reg, info); 233 } 234 235 236 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 237 Metadata* o = (Metadata*)Universe::non_oop_word(); 238 int index = __ oop_recorder()->allocate_metadata_index(o); 239 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index); 240 241 __ patchable_mov_metadata(reg, o, index); 242 patching_epilog(patch, lir_patch_normal, reg, info); 243 } 244 245 246 int LIR_Assembler::initial_frame_size_in_bytes() const { 247 // Subtracts two words to account for return address and link 248 return frame_map()->framesize()*VMRegImpl::stack_slot_size - 2*wordSize; 249 } 250 251 252 int LIR_Assembler::emit_exception_handler() { 253 // TODO: ARM 254 __ nop(); // See comments in other ports 255 256 address handler_base = __ start_a_stub(exception_handler_size()); 257 if (handler_base == NULL) { 258 bailout("exception handler overflow"); 259 return -1; 260 } 261 262 int offset = code_offset(); 263 264 // check that there is really an exception 265 __ verify_not_null_oop(Rexception_obj); 266 267 __ call(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id), relocInfo::runtime_call_type); 268 __ should_not_reach_here(); 269 270 assert(code_offset() - offset <= exception_handler_size(), "overflow"); 271 __ end_a_stub(); 272 273 return offset; 274 } 275 276 // Emit the code to remove the frame from the stack in the exception 277 // unwind path. 278 int LIR_Assembler::emit_unwind_handler() { 279 #ifndef PRODUCT 280 if (CommentedAssembly) { 281 _masm->block_comment("Unwind handler"); 282 } 283 #endif 284 285 int offset = code_offset(); 286 287 // Fetch the exception from TLS and clear out exception related thread state 288 Register zero = __ zero_register(Rtemp); 289 __ ldr(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset())); 290 __ str(zero, Address(Rthread, JavaThread::exception_oop_offset())); 291 __ str(zero, Address(Rthread, JavaThread::exception_pc_offset())); 292 293 __ bind(_unwind_handler_entry); 294 __ verify_not_null_oop(Rexception_obj); 295 296 // Preform needed unlocking 297 MonitorExitStub* stub = NULL; 298 if (method()->is_synchronized()) { 299 monitor_address(0, FrameMap::R0_opr); 300 stub = new MonitorExitStub(FrameMap::R0_opr, true, 0); 301 __ unlock_object(R2, R1, R0, Rtemp, *stub->entry()); 302 __ bind(*stub->continuation()); 303 } 304 305 // remove the activation and dispatch to the unwind handler 306 __ remove_frame(initial_frame_size_in_bytes()); // restores FP and LR 307 __ jump(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type, Rtemp); 308 309 // Emit the slow path assembly 310 if (stub != NULL) { 311 stub->emit_code(this); 312 } 313 314 return offset; 315 } 316 317 318 int LIR_Assembler::emit_deopt_handler() { 319 address handler_base = __ start_a_stub(deopt_handler_size()); 320 if (handler_base == NULL) { 321 bailout("deopt handler overflow"); 322 return -1; 323 } 324 325 int offset = code_offset(); 326 327 __ mov_relative_address(LR, __ pc()); 328 #ifdef AARCH64 329 __ raw_push(LR, LR); 330 __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, Rtemp); 331 #else 332 __ push(LR); // stub expects LR to be saved 333 __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg); 334 #endif // AARCH64 335 336 assert(code_offset() - offset <= deopt_handler_size(), "overflow"); 337 __ end_a_stub(); 338 339 return offset; 340 } 341 342 343 void LIR_Assembler::return_op(LIR_Opr result) { 344 // Pop the frame before safepoint polling 345 __ remove_frame(initial_frame_size_in_bytes()); 346 347 // mov_slow here is usually one or two instruction 348 // TODO-AARCH64 3 instructions on AArch64, so try to load polling page by ldr_literal 349 __ mov_address(Rtemp, os::get_polling_page(), symbolic_Relocation::polling_page_reference); 350 __ relocate(relocInfo::poll_return_type); 351 __ ldr(Rtemp, Address(Rtemp)); 352 __ ret(); 353 } 354 355 356 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 357 __ mov_address(Rtemp, os::get_polling_page(), symbolic_Relocation::polling_page_reference); 358 if (info != NULL) { 359 add_debug_info_for_branch(info); 360 } 361 int offset = __ offset(); 362 __ relocate(relocInfo::poll_type); 363 __ ldr(Rtemp, Address(Rtemp)); 364 return offset; 365 } 366 367 368 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 369 if (from_reg != to_reg) { 370 __ mov(to_reg, from_reg); 371 } 372 } 373 374 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 375 assert(src->is_constant() && dest->is_register(), "must be"); 376 LIR_Const* c = src->as_constant_ptr(); 377 378 switch (c->type()) { 379 case T_ADDRESS: 380 case T_INT: 381 assert(patch_code == lir_patch_none, "no patching handled here"); 382 __ mov_slow(dest->as_register(), c->as_jint()); 383 break; 384 385 case T_LONG: 386 assert(patch_code == lir_patch_none, "no patching handled here"); 387 #ifdef AARCH64 388 __ mov_slow(dest->as_pointer_register(), (intptr_t)c->as_jlong()); 389 #else 390 __ mov_slow(dest->as_register_lo(), c->as_jint_lo()); 391 __ mov_slow(dest->as_register_hi(), c->as_jint_hi()); 392 #endif // AARCH64 393 break; 394 395 case T_OBJECT: 396 if (patch_code == lir_patch_none) { 397 __ mov_oop(dest->as_register(), c->as_jobject()); 398 } else { 399 jobject2reg_with_patching(dest->as_register(), info); 400 } 401 break; 402 403 case T_METADATA: 404 if (patch_code == lir_patch_none) { 405 __ mov_metadata(dest->as_register(), c->as_metadata()); 406 } else { 407 klass2reg_with_patching(dest->as_register(), info); 408 } 409 break; 410 411 case T_FLOAT: 412 if (dest->is_single_fpu()) { 413 __ mov_float(dest->as_float_reg(), c->as_jfloat()); 414 } else { 415 #ifdef AARCH64 416 ShouldNotReachHere(); 417 #else 418 // Simple getters can return float constant directly into r0 419 __ mov_slow(dest->as_register(), c->as_jint_bits()); 420 #endif // AARCH64 421 } 422 break; 423 424 case T_DOUBLE: 425 if (dest->is_double_fpu()) { 426 __ mov_double(dest->as_double_reg(), c->as_jdouble()); 427 } else { 428 #ifdef AARCH64 429 ShouldNotReachHere(); 430 #else 431 // Simple getters can return double constant directly into r1r0 432 __ mov_slow(dest->as_register_lo(), c->as_jint_lo_bits()); 433 __ mov_slow(dest->as_register_hi(), c->as_jint_hi_bits()); 434 #endif // AARCH64 435 } 436 break; 437 438 default: 439 ShouldNotReachHere(); 440 } 441 } 442 443 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 444 assert(src->is_constant(), "must be"); 445 assert(dest->is_stack(), "must be"); 446 LIR_Const* c = src->as_constant_ptr(); 447 448 switch (c->type()) { 449 case T_INT: // fall through 450 case T_FLOAT: 451 __ mov_slow(Rtemp, c->as_jint_bits()); 452 __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 453 break; 454 455 case T_ADDRESS: 456 __ mov_slow(Rtemp, c->as_jint()); 457 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 458 break; 459 460 case T_OBJECT: 461 __ mov_oop(Rtemp, c->as_jobject()); 462 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 463 break; 464 465 case T_LONG: // fall through 466 case T_DOUBLE: 467 #ifdef AARCH64 468 __ mov_slow(Rtemp, c->as_jlong_bits()); 469 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix())); 470 #else 471 __ mov_slow(Rtemp, c->as_jint_lo_bits()); 472 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes)); 473 if (c->as_jint_hi_bits() != c->as_jint_lo_bits()) { 474 __ mov_slow(Rtemp, c->as_jint_hi_bits()); 475 } 476 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes)); 477 #endif // AARCH64 478 break; 479 480 default: 481 ShouldNotReachHere(); 482 } 483 } 484 485 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, 486 CodeEmitInfo* info, bool wide) { 487 #ifdef AARCH64 488 assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL) || 489 (src->as_constant_ptr()->type() == T_INT && src->as_constant_ptr()->as_jint() == 0) || 490 (src->as_constant_ptr()->type() == T_LONG && src->as_constant_ptr()->as_jlong() == 0) || 491 (src->as_constant_ptr()->type() == T_FLOAT && src->as_constant_ptr()->as_jint_bits() == 0) || 492 (src->as_constant_ptr()->type() == T_DOUBLE && src->as_constant_ptr()->as_jlong_bits() == 0), 493 "cannot handle otherwise"); 494 assert(dest->as_address_ptr()->type() == type, "should be"); 495 496 Address addr = as_Address(dest->as_address_ptr()); 497 int null_check_offset = code_offset(); 498 switch (type) { 499 case T_OBJECT: // fall through 500 case T_ARRAY: 501 if (UseCompressedOops && !wide) { 502 __ str_w(ZR, addr); 503 } else { 504 __ str(ZR, addr); 505 } 506 break; 507 case T_ADDRESS: // fall through 508 case T_DOUBLE: // fall through 509 case T_LONG: __ str(ZR, addr); break; 510 case T_FLOAT: // fall through 511 case T_INT: __ str_w(ZR, addr); break; 512 case T_BOOLEAN: // fall through 513 case T_BYTE: __ strb(ZR, addr); break; 514 case T_CHAR: // fall through 515 case T_SHORT: __ strh(ZR, addr); break; 516 default: ShouldNotReachHere(); 517 } 518 #else 519 assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL),"cannot handle otherwise"); 520 __ mov(Rtemp, 0); 521 522 int null_check_offset = code_offset(); 523 __ str(Rtemp, as_Address(dest->as_address_ptr())); 524 #endif // AARCH64 525 526 if (info != NULL) { 527 #ifndef AARCH64 528 assert(false, "arm32 didn't support this before, investigate if bug"); 529 #endif 530 add_debug_info_for_null_check(null_check_offset, info); 531 } 532 } 533 534 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 535 assert(src->is_register() && dest->is_register(), "must be"); 536 537 if (src->is_single_cpu()) { 538 if (dest->is_single_cpu()) { 539 move_regs(src->as_register(), dest->as_register()); 540 #ifdef AARCH64 541 } else if (dest->is_double_cpu()) { 542 assert ((src->type() == T_OBJECT) || (src->type() == T_ARRAY) || (src->type() == T_ADDRESS), "invalid src type"); 543 move_regs(src->as_register(), dest->as_register_lo()); 544 #else 545 } else if (dest->is_single_fpu()) { 546 __ fmsr(dest->as_float_reg(), src->as_register()); 547 #endif // AARCH64 548 } else { 549 ShouldNotReachHere(); 550 } 551 } else if (src->is_double_cpu()) { 552 #ifdef AARCH64 553 move_regs(src->as_register_lo(), dest->as_register_lo()); 554 #else 555 if (dest->is_double_cpu()) { 556 __ long_move(dest->as_register_lo(), dest->as_register_hi(), src->as_register_lo(), src->as_register_hi()); 557 } else { 558 __ fmdrr(dest->as_double_reg(), src->as_register_lo(), src->as_register_hi()); 559 } 560 #endif // AARCH64 561 } else if (src->is_single_fpu()) { 562 if (dest->is_single_fpu()) { 563 __ mov_float(dest->as_float_reg(), src->as_float_reg()); 564 } else if (dest->is_single_cpu()) { 565 __ mov_fpr2gpr_float(dest->as_register(), src->as_float_reg()); 566 } else { 567 ShouldNotReachHere(); 568 } 569 } else if (src->is_double_fpu()) { 570 if (dest->is_double_fpu()) { 571 __ mov_double(dest->as_double_reg(), src->as_double_reg()); 572 } else if (dest->is_double_cpu()) { 573 #ifdef AARCH64 574 __ fmov_xd(dest->as_register_lo(), src->as_double_reg()); 575 #else 576 __ fmrrd(dest->as_register_lo(), dest->as_register_hi(), src->as_double_reg()); 577 #endif // AARCH64 578 } else { 579 ShouldNotReachHere(); 580 } 581 } else { 582 ShouldNotReachHere(); 583 } 584 } 585 586 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 587 assert(src->is_register(), "should not call otherwise"); 588 assert(dest->is_stack(), "should not call otherwise"); 589 590 Address addr = dest->is_single_word() ? 591 frame_map()->address_for_slot(dest->single_stack_ix()) : 592 frame_map()->address_for_slot(dest->double_stack_ix()); 593 594 #ifndef AARCH64 595 assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending"); 596 if (src->is_single_fpu() || src->is_double_fpu()) { 597 if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); } 598 } 599 #endif // !AARCH64 600 601 if (src->is_single_cpu()) { 602 switch (type) { 603 case T_OBJECT: 604 case T_ARRAY: __ verify_oop(src->as_register()); // fall through 605 case T_ADDRESS: 606 case T_METADATA: __ str(src->as_register(), addr); break; 607 case T_FLOAT: // used in intBitsToFloat intrinsic implementation, fall through 608 case T_INT: __ str_32(src->as_register(), addr); break; 609 default: 610 ShouldNotReachHere(); 611 } 612 } else if (src->is_double_cpu()) { 613 __ str(src->as_register_lo(), addr); 614 #ifndef AARCH64 615 __ str(src->as_register_hi(), frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes)); 616 #endif // !AARCH64 617 } else if (src->is_single_fpu()) { 618 __ str_float(src->as_float_reg(), addr); 619 } else if (src->is_double_fpu()) { 620 __ str_double(src->as_double_reg(), addr); 621 } else { 622 ShouldNotReachHere(); 623 } 624 } 625 626 627 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, 628 LIR_PatchCode patch_code, CodeEmitInfo* info, 629 bool pop_fpu_stack, bool wide, 630 bool unaligned) { 631 LIR_Address* to_addr = dest->as_address_ptr(); 632 Register base_reg = to_addr->base()->as_pointer_register(); 633 const bool needs_patching = (patch_code != lir_patch_none); 634 635 PatchingStub* patch = NULL; 636 if (needs_patching) { 637 #ifdef AARCH64 638 // Same alignment of reg2mem code and PatchingStub code. Required to make copied bind_literal() code properly aligned. 639 __ align(wordSize); 640 #endif 641 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 642 #ifdef AARCH64 643 // Extra nop for MT safe patching 644 __ nop(); 645 #endif // AARCH64 646 } 647 648 int null_check_offset = code_offset(); 649 650 switch (type) { 651 case T_ARRAY: 652 case T_OBJECT: 653 if (UseCompressedOops && !wide) { 654 #ifdef AARCH64 655 const Register temp_src = Rtemp; 656 assert_different_registers(temp_src, src->as_register()); 657 __ encode_heap_oop(temp_src, src->as_register()); 658 null_check_offset = code_offset(); 659 __ str_32(temp_src, as_Address(to_addr)); 660 #else 661 ShouldNotReachHere(); 662 #endif // AARCH64 663 } else { 664 __ str(src->as_register(), as_Address(to_addr)); 665 } 666 break; 667 668 case T_ADDRESS: 669 #ifdef AARCH64 670 case T_LONG: 671 #endif // AARCH64 672 __ str(src->as_pointer_register(), as_Address(to_addr)); 673 break; 674 675 case T_BYTE: 676 case T_BOOLEAN: 677 __ strb(src->as_register(), as_Address(to_addr)); 678 break; 679 680 case T_CHAR: 681 case T_SHORT: 682 __ strh(src->as_register(), as_Address(to_addr)); 683 break; 684 685 case T_INT: 686 #ifdef __SOFTFP__ 687 case T_FLOAT: 688 #endif // __SOFTFP__ 689 __ str_32(src->as_register(), as_Address(to_addr)); 690 break; 691 692 #ifdef AARCH64 693 694 case T_FLOAT: 695 __ str_s(src->as_float_reg(), as_Address(to_addr)); 696 break; 697 698 case T_DOUBLE: 699 __ str_d(src->as_double_reg(), as_Address(to_addr)); 700 break; 701 702 #else // AARCH64 703 704 #ifdef __SOFTFP__ 705 case T_DOUBLE: 706 #endif // __SOFTFP__ 707 case T_LONG: { 708 Register from_lo = src->as_register_lo(); 709 Register from_hi = src->as_register_hi(); 710 if (to_addr->index()->is_register()) { 711 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 712 assert(to_addr->disp() == 0, "Not yet supporting both"); 713 __ add(Rtemp, base_reg, to_addr->index()->as_register()); 714 base_reg = Rtemp; 715 __ str(from_lo, Address(Rtemp)); 716 if (patch != NULL) { 717 patching_epilog(patch, lir_patch_low, base_reg, info); 718 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 719 patch_code = lir_patch_high; 720 } 721 __ str(from_hi, Address(Rtemp, BytesPerWord)); 722 } else if (base_reg == from_lo) { 723 __ str(from_hi, as_Address_hi(to_addr)); 724 if (patch != NULL) { 725 patching_epilog(patch, lir_patch_high, base_reg, info); 726 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 727 patch_code = lir_patch_low; 728 } 729 __ str(from_lo, as_Address_lo(to_addr)); 730 } else { 731 __ str(from_lo, as_Address_lo(to_addr)); 732 if (patch != NULL) { 733 patching_epilog(patch, lir_patch_low, base_reg, info); 734 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 735 patch_code = lir_patch_high; 736 } 737 __ str(from_hi, as_Address_hi(to_addr)); 738 } 739 break; 740 } 741 742 #ifndef __SOFTFP__ 743 case T_FLOAT: 744 if (to_addr->index()->is_register()) { 745 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 746 __ add(Rtemp, base_reg, to_addr->index()->as_register()); 747 if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); } 748 __ fsts(src->as_float_reg(), Address(Rtemp, to_addr->disp())); 749 } else { 750 __ fsts(src->as_float_reg(), as_Address(to_addr)); 751 } 752 break; 753 754 case T_DOUBLE: 755 if (to_addr->index()->is_register()) { 756 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 757 __ add(Rtemp, base_reg, to_addr->index()->as_register()); 758 if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); } 759 __ fstd(src->as_double_reg(), Address(Rtemp, to_addr->disp())); 760 } else { 761 __ fstd(src->as_double_reg(), as_Address(to_addr)); 762 } 763 break; 764 #endif // __SOFTFP__ 765 766 #endif // AARCH64 767 768 default: 769 ShouldNotReachHere(); 770 } 771 772 if (info != NULL) { 773 add_debug_info_for_null_check(null_check_offset, info); 774 } 775 776 if (patch != NULL) { 777 // Offset embeedded into LDR/STR instruction may appear not enough 778 // to address a field. So, provide a space for one more instruction 779 // that will deal with larger offsets. 780 __ nop(); 781 patching_epilog(patch, patch_code, base_reg, info); 782 } 783 } 784 785 786 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 787 assert(src->is_stack(), "should not call otherwise"); 788 assert(dest->is_register(), "should not call otherwise"); 789 790 Address addr = src->is_single_word() ? 791 frame_map()->address_for_slot(src->single_stack_ix()) : 792 frame_map()->address_for_slot(src->double_stack_ix()); 793 794 #ifndef AARCH64 795 assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending"); 796 if (dest->is_single_fpu() || dest->is_double_fpu()) { 797 if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); } 798 } 799 #endif // !AARCH64 800 801 if (dest->is_single_cpu()) { 802 switch (type) { 803 case T_OBJECT: 804 case T_ARRAY: 805 case T_ADDRESS: 806 case T_METADATA: __ ldr(dest->as_register(), addr); break; 807 case T_FLOAT: // used in floatToRawIntBits intrinsic implemenation 808 case T_INT: __ ldr_u32(dest->as_register(), addr); break; 809 default: 810 ShouldNotReachHere(); 811 } 812 if ((type == T_OBJECT) || (type == T_ARRAY)) { 813 __ verify_oop(dest->as_register()); 814 } 815 } else if (dest->is_double_cpu()) { 816 __ ldr(dest->as_register_lo(), addr); 817 #ifndef AARCH64 818 __ ldr(dest->as_register_hi(), frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes)); 819 #endif // !AARCH64 820 } else if (dest->is_single_fpu()) { 821 __ ldr_float(dest->as_float_reg(), addr); 822 } else if (dest->is_double_fpu()) { 823 __ ldr_double(dest->as_double_reg(), addr); 824 } else { 825 ShouldNotReachHere(); 826 } 827 } 828 829 830 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 831 if (src->is_single_stack()) { 832 switch (src->type()) { 833 case T_OBJECT: 834 case T_ARRAY: 835 case T_ADDRESS: 836 case T_METADATA: 837 __ ldr(Rtemp, frame_map()->address_for_slot(src->single_stack_ix())); 838 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 839 break; 840 841 case T_INT: 842 case T_FLOAT: 843 __ ldr_u32(Rtemp, frame_map()->address_for_slot(src->single_stack_ix())); 844 __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 845 break; 846 847 default: 848 ShouldNotReachHere(); 849 } 850 } else { 851 assert(src->is_double_stack(), "must be"); 852 __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes)); 853 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes)); 854 #ifdef AARCH64 855 assert(lo_word_offset_in_bytes == 0, "adjust this code"); 856 #else 857 __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes)); 858 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes)); 859 #endif // AARCH64 860 } 861 } 862 863 864 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, 865 LIR_PatchCode patch_code, CodeEmitInfo* info, 866 bool wide, bool unaligned) { 867 assert(src->is_address(), "should not call otherwise"); 868 assert(dest->is_register(), "should not call otherwise"); 869 LIR_Address* addr = src->as_address_ptr(); 870 871 Register base_reg = addr->base()->as_pointer_register(); 872 873 PatchingStub* patch = NULL; 874 if (patch_code != lir_patch_none) { 875 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 876 #ifdef AARCH64 877 // Extra nop for MT safe patching 878 __ nop(); 879 #endif // AARCH64 880 } 881 if (info != NULL) { 882 add_debug_info_for_null_check_here(info); 883 } 884 885 switch (type) { 886 case T_OBJECT: // fall through 887 case T_ARRAY: 888 if (UseCompressedOops && !wide) { 889 __ ldr_u32(dest->as_register(), as_Address(addr)); 890 } else { 891 __ ldr(dest->as_register(), as_Address(addr)); 892 } 893 break; 894 895 case T_ADDRESS: 896 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { 897 __ ldr_u32(dest->as_pointer_register(), as_Address(addr)); 898 } else { 899 __ ldr(dest->as_pointer_register(), as_Address(addr)); 900 } 901 break; 902 903 #ifdef AARCH64 904 case T_LONG: 905 #else 906 case T_INT: 907 #ifdef __SOFTFP__ 908 case T_FLOAT: 909 #endif // __SOFTFP__ 910 #endif // AARCH64 911 __ ldr(dest->as_pointer_register(), as_Address(addr)); 912 break; 913 914 case T_BOOLEAN: 915 __ ldrb(dest->as_register(), as_Address(addr)); 916 break; 917 918 case T_BYTE: 919 __ ldrsb(dest->as_register(), as_Address(addr)); 920 break; 921 922 case T_CHAR: 923 __ ldrh(dest->as_register(), as_Address(addr)); 924 break; 925 926 case T_SHORT: 927 __ ldrsh(dest->as_register(), as_Address(addr)); 928 break; 929 930 #ifdef AARCH64 931 932 case T_INT: 933 __ ldr_w(dest->as_register(), as_Address(addr)); 934 break; 935 936 case T_FLOAT: 937 __ ldr_s(dest->as_float_reg(), as_Address(addr)); 938 break; 939 940 case T_DOUBLE: 941 __ ldr_d(dest->as_double_reg(), as_Address(addr)); 942 break; 943 944 #else // AARCH64 945 946 #ifdef __SOFTFP__ 947 case T_DOUBLE: 948 #endif // __SOFTFP__ 949 case T_LONG: { 950 Register to_lo = dest->as_register_lo(); 951 Register to_hi = dest->as_register_hi(); 952 if (addr->index()->is_register()) { 953 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 954 assert(addr->disp() == 0, "Not yet supporting both"); 955 __ add(Rtemp, base_reg, addr->index()->as_register()); 956 base_reg = Rtemp; 957 __ ldr(to_lo, Address(Rtemp)); 958 if (patch != NULL) { 959 patching_epilog(patch, lir_patch_low, base_reg, info); 960 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 961 patch_code = lir_patch_high; 962 } 963 __ ldr(to_hi, Address(Rtemp, BytesPerWord)); 964 } else if (base_reg == to_lo) { 965 __ ldr(to_hi, as_Address_hi(addr)); 966 if (patch != NULL) { 967 patching_epilog(patch, lir_patch_high, base_reg, info); 968 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 969 patch_code = lir_patch_low; 970 } 971 __ ldr(to_lo, as_Address_lo(addr)); 972 } else { 973 __ ldr(to_lo, as_Address_lo(addr)); 974 if (patch != NULL) { 975 patching_epilog(patch, lir_patch_low, base_reg, info); 976 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 977 patch_code = lir_patch_high; 978 } 979 __ ldr(to_hi, as_Address_hi(addr)); 980 } 981 break; 982 } 983 984 #ifndef __SOFTFP__ 985 case T_FLOAT: 986 if (addr->index()->is_register()) { 987 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 988 __ add(Rtemp, base_reg, addr->index()->as_register()); 989 if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); } 990 __ flds(dest->as_float_reg(), Address(Rtemp, addr->disp())); 991 } else { 992 __ flds(dest->as_float_reg(), as_Address(addr)); 993 } 994 break; 995 996 case T_DOUBLE: 997 if (addr->index()->is_register()) { 998 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 999 __ add(Rtemp, base_reg, addr->index()->as_register()); 1000 if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); } 1001 __ fldd(dest->as_double_reg(), Address(Rtemp, addr->disp())); 1002 } else { 1003 __ fldd(dest->as_double_reg(), as_Address(addr)); 1004 } 1005 break; 1006 #endif // __SOFTFP__ 1007 1008 #endif // AARCH64 1009 1010 default: 1011 ShouldNotReachHere(); 1012 } 1013 1014 if (patch != NULL) { 1015 // Offset embeedded into LDR/STR instruction may appear not enough 1016 // to address a field. So, provide a space for one more instruction 1017 // that will deal with larger offsets. 1018 __ nop(); 1019 patching_epilog(patch, patch_code, base_reg, info); 1020 } 1021 1022 #ifdef AARCH64 1023 switch (type) { 1024 case T_ARRAY: 1025 case T_OBJECT: 1026 if (UseCompressedOops && !wide) { 1027 __ decode_heap_oop(dest->as_register()); 1028 } 1029 __ verify_oop(dest->as_register()); 1030 break; 1031 1032 case T_ADDRESS: 1033 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1034 __ decode_klass_not_null(dest->as_register()); 1035 } 1036 break; 1037 } 1038 #endif // AARCH64 1039 } 1040 1041 1042 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1043 bool is_32 = op->result_opr()->is_single_cpu(); 1044 1045 if (op->code() == lir_idiv && op->in_opr2()->is_constant() && is_32) { 1046 int c = op->in_opr2()->as_constant_ptr()->as_jint(); 1047 assert(is_power_of_2(c), "non power-of-2 constant should be put in a register"); 1048 1049 Register left = op->in_opr1()->as_register(); 1050 Register dest = op->result_opr()->as_register(); 1051 if (c == 1) { 1052 __ mov(dest, left); 1053 } else if (c == 2) { 1054 __ add_32(dest, left, AsmOperand(left, lsr, 31)); 1055 __ asr_32(dest, dest, 1); 1056 } else if (c != (int) 0x80000000) { 1057 int power = log2_intptr(c); 1058 __ asr_32(Rtemp, left, 31); 1059 __ add_32(dest, left, AsmOperand(Rtemp, lsr, 32-power)); // dest = left + (left < 0 ? 2^power - 1 : 0); 1060 __ asr_32(dest, dest, power); // dest = dest >>> power; 1061 } else { 1062 // x/0x80000000 is a special case, since dividend is a power of two, but is negative. 1063 // The only possible result values are 0 and 1, with 1 only for dividend == divisor == 0x80000000. 1064 __ cmp_32(left, c); 1065 #ifdef AARCH64 1066 __ cset(dest, eq); 1067 #else 1068 __ mov(dest, 0, ne); 1069 __ mov(dest, 1, eq); 1070 #endif // AARCH64 1071 } 1072 } else { 1073 #ifdef AARCH64 1074 Register left = op->in_opr1()->as_pointer_register(); 1075 Register right = op->in_opr2()->as_pointer_register(); 1076 Register dest = op->result_opr()->as_pointer_register(); 1077 1078 switch (op->code()) { 1079 case lir_idiv: 1080 if (is_32) { 1081 __ sdiv_w(dest, left, right); 1082 } else { 1083 __ sdiv(dest, left, right); 1084 } 1085 break; 1086 case lir_irem: { 1087 Register tmp = op->in_opr3()->as_pointer_register(); 1088 assert_different_registers(left, tmp); 1089 assert_different_registers(right, tmp); 1090 if (is_32) { 1091 __ sdiv_w(tmp, left, right); 1092 __ msub_w(dest, right, tmp, left); 1093 } else { 1094 __ sdiv(tmp, left, right); 1095 __ msub(dest, right, tmp, left); 1096 } 1097 break; 1098 } 1099 default: 1100 ShouldNotReachHere(); 1101 } 1102 #else 1103 assert(op->code() == lir_idiv || op->code() == lir_irem, "unexpected op3"); 1104 __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type); 1105 add_debug_info_for_div0_here(op->info()); 1106 #endif // AARCH64 1107 } 1108 } 1109 1110 1111 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1112 #ifdef ASSERT 1113 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label"); 1114 if (op->block() != NULL) _branch_target_blocks.append(op->block()); 1115 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock()); 1116 assert(op->info() == NULL, "CodeEmitInfo?"); 1117 #endif // ASSERT 1118 1119 #ifdef __SOFTFP__ 1120 assert (op->code() != lir_cond_float_branch, "this should be impossible"); 1121 #else 1122 if (op->code() == lir_cond_float_branch) { 1123 #ifndef AARCH64 1124 __ fmstat(); 1125 #endif // !AARCH64 1126 __ b(*(op->ublock()->label()), vs); 1127 } 1128 #endif // __SOFTFP__ 1129 1130 AsmCondition acond = al; 1131 switch (op->cond()) { 1132 case lir_cond_equal: acond = eq; break; 1133 case lir_cond_notEqual: acond = ne; break; 1134 case lir_cond_less: acond = lt; break; 1135 case lir_cond_lessEqual: acond = le; break; 1136 case lir_cond_greaterEqual: acond = ge; break; 1137 case lir_cond_greater: acond = gt; break; 1138 case lir_cond_aboveEqual: acond = hs; break; 1139 case lir_cond_belowEqual: acond = ls; break; 1140 default: assert(op->cond() == lir_cond_always, "must be"); 1141 } 1142 __ b(*(op->label()), acond); 1143 } 1144 1145 1146 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1147 LIR_Opr src = op->in_opr(); 1148 LIR_Opr dest = op->result_opr(); 1149 1150 switch (op->bytecode()) { 1151 case Bytecodes::_i2l: 1152 #ifdef AARCH64 1153 __ sign_extend(dest->as_register_lo(), src->as_register(), 32); 1154 #else 1155 move_regs(src->as_register(), dest->as_register_lo()); 1156 __ mov(dest->as_register_hi(), AsmOperand(src->as_register(), asr, 31)); 1157 #endif // AARCH64 1158 break; 1159 case Bytecodes::_l2i: 1160 move_regs(src->as_register_lo(), dest->as_register()); 1161 break; 1162 case Bytecodes::_i2b: 1163 __ sign_extend(dest->as_register(), src->as_register(), 8); 1164 break; 1165 case Bytecodes::_i2s: 1166 __ sign_extend(dest->as_register(), src->as_register(), 16); 1167 break; 1168 case Bytecodes::_i2c: 1169 __ zero_extend(dest->as_register(), src->as_register(), 16); 1170 break; 1171 case Bytecodes::_f2d: 1172 __ convert_f2d(dest->as_double_reg(), src->as_float_reg()); 1173 break; 1174 case Bytecodes::_d2f: 1175 __ convert_d2f(dest->as_float_reg(), src->as_double_reg()); 1176 break; 1177 case Bytecodes::_i2f: 1178 #ifdef AARCH64 1179 __ scvtf_sw(dest->as_float_reg(), src->as_register()); 1180 #else 1181 __ fmsr(Stemp, src->as_register()); 1182 __ fsitos(dest->as_float_reg(), Stemp); 1183 #endif // AARCH64 1184 break; 1185 case Bytecodes::_i2d: 1186 #ifdef AARCH64 1187 __ scvtf_dw(dest->as_double_reg(), src->as_register()); 1188 #else 1189 __ fmsr(Stemp, src->as_register()); 1190 __ fsitod(dest->as_double_reg(), Stemp); 1191 #endif // AARCH64 1192 break; 1193 case Bytecodes::_f2i: 1194 #ifdef AARCH64 1195 __ fcvtzs_ws(dest->as_register(), src->as_float_reg()); 1196 #else 1197 __ ftosizs(Stemp, src->as_float_reg()); 1198 __ fmrs(dest->as_register(), Stemp); 1199 #endif // AARCH64 1200 break; 1201 case Bytecodes::_d2i: 1202 #ifdef AARCH64 1203 __ fcvtzs_wd(dest->as_register(), src->as_double_reg()); 1204 #else 1205 __ ftosizd(Stemp, src->as_double_reg()); 1206 __ fmrs(dest->as_register(), Stemp); 1207 #endif // AARCH64 1208 break; 1209 #ifdef AARCH64 1210 case Bytecodes::_l2f: 1211 __ scvtf_sx(dest->as_float_reg(), src->as_register_lo()); 1212 break; 1213 case Bytecodes::_l2d: 1214 __ scvtf_dx(dest->as_double_reg(), src->as_register_lo()); 1215 break; 1216 case Bytecodes::_f2l: 1217 __ fcvtzs_xs(dest->as_register_lo(), src->as_float_reg()); 1218 break; 1219 case Bytecodes::_d2l: 1220 __ fcvtzs_xd(dest->as_register_lo(), src->as_double_reg()); 1221 break; 1222 #endif // AARCH64 1223 default: 1224 ShouldNotReachHere(); 1225 } 1226 } 1227 1228 1229 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1230 if (op->init_check()) { 1231 Register tmp = op->tmp1()->as_register(); 1232 __ ldrb(tmp, Address(op->klass()->as_register(), InstanceKlass::init_state_offset())); 1233 add_debug_info_for_null_check_here(op->stub()->info()); 1234 __ cmp(tmp, InstanceKlass::fully_initialized); 1235 __ b(*op->stub()->entry(), ne); 1236 } 1237 __ allocate_object(op->obj()->as_register(), 1238 op->tmp1()->as_register(), 1239 op->tmp2()->as_register(), 1240 op->tmp3()->as_register(), 1241 op->header_size(), 1242 op->object_size(), 1243 op->klass()->as_register(), 1244 *op->stub()->entry()); 1245 __ bind(*op->stub()->continuation()); 1246 } 1247 1248 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1249 if (UseSlowPath || 1250 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || 1251 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { 1252 __ b(*op->stub()->entry()); 1253 } else { 1254 __ allocate_array(op->obj()->as_register(), 1255 op->len()->as_register(), 1256 op->tmp1()->as_register(), 1257 op->tmp2()->as_register(), 1258 op->tmp3()->as_register(), 1259 arrayOopDesc::header_size(op->type()), 1260 type2aelembytes(op->type()), 1261 op->klass()->as_register(), 1262 *op->stub()->entry()); 1263 } 1264 __ bind(*op->stub()->continuation()); 1265 } 1266 1267 void LIR_Assembler::type_profile_helper(Register mdo, int mdo_offset_bias, 1268 ciMethodData *md, ciProfileData *data, 1269 Register recv, Register tmp1, Label* update_done) { 1270 assert_different_registers(mdo, recv, tmp1); 1271 uint i; 1272 for (i = 0; i < VirtualCallData::row_limit(); i++) { 1273 Label next_test; 1274 // See if the receiver is receiver[n]. 1275 Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) - 1276 mdo_offset_bias); 1277 __ ldr(tmp1, receiver_addr); 1278 __ verify_klass_ptr(tmp1); 1279 __ cmp(recv, tmp1); 1280 __ b(next_test, ne); 1281 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) - 1282 mdo_offset_bias); 1283 __ ldr(tmp1, data_addr); 1284 __ add(tmp1, tmp1, DataLayout::counter_increment); 1285 __ str(tmp1, data_addr); 1286 __ b(*update_done); 1287 __ bind(next_test); 1288 } 1289 1290 // Didn't find receiver; find next empty slot and fill it in 1291 for (i = 0; i < VirtualCallData::row_limit(); i++) { 1292 Label next_test; 1293 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) - 1294 mdo_offset_bias); 1295 __ ldr(tmp1, recv_addr); 1296 __ cbnz(tmp1, next_test); 1297 __ str(recv, recv_addr); 1298 __ mov(tmp1, DataLayout::counter_increment); 1299 __ str(tmp1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) - 1300 mdo_offset_bias)); 1301 __ b(*update_done); 1302 __ bind(next_test); 1303 } 1304 } 1305 1306 void LIR_Assembler::setup_md_access(ciMethod* method, int bci, 1307 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) { 1308 md = method->method_data_or_null(); 1309 assert(md != NULL, "Sanity"); 1310 data = md->bci_to_data(bci); 1311 assert(data != NULL, "need data for checkcast"); 1312 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1313 if (md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes() >= 4096) { 1314 // The offset is large so bias the mdo by the base of the slot so 1315 // that the ldr can use an immediate offset to reference the slots of the data 1316 mdo_offset_bias = md->byte_offset_of_slot(data, DataLayout::header_offset()); 1317 } 1318 } 1319 1320 // On 32-bit ARM, code before this helper should test obj for null (ZF should be set if obj is null). 1321 void LIR_Assembler::typecheck_profile_helper1(ciMethod* method, int bci, 1322 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias, 1323 Register obj, Register mdo, Register data_val, Label* obj_is_null) { 1324 assert(method != NULL, "Should have method"); 1325 assert_different_registers(obj, mdo, data_val); 1326 setup_md_access(method, bci, md, data, mdo_offset_bias); 1327 Label not_null; 1328 #ifdef AARCH64 1329 __ cbnz(obj, not_null); 1330 #else 1331 __ b(not_null, ne); 1332 #endif // AARCH64 1333 __ mov_metadata(mdo, md->constant_encoding()); 1334 if (mdo_offset_bias > 0) { 1335 __ mov_slow(data_val, mdo_offset_bias); 1336 __ add(mdo, mdo, data_val); 1337 } 1338 Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias); 1339 __ ldrb(data_val, flags_addr); 1340 __ orr(data_val, data_val, (uint)BitData::null_seen_byte_constant()); 1341 __ strb(data_val, flags_addr); 1342 __ b(*obj_is_null); 1343 __ bind(not_null); 1344 } 1345 1346 void LIR_Assembler::typecheck_profile_helper2(ciMethodData* md, ciProfileData* data, int mdo_offset_bias, 1347 Register mdo, Register recv, Register value, Register tmp1, 1348 Label* profile_cast_success, Label* profile_cast_failure, 1349 Label* success, Label* failure) { 1350 assert_different_registers(mdo, value, tmp1); 1351 __ bind(*profile_cast_success); 1352 __ mov_metadata(mdo, md->constant_encoding()); 1353 if (mdo_offset_bias > 0) { 1354 __ mov_slow(tmp1, mdo_offset_bias); 1355 __ add(mdo, mdo, tmp1); 1356 } 1357 __ load_klass(recv, value); 1358 type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, success); 1359 __ b(*success); 1360 // Cast failure case 1361 __ bind(*profile_cast_failure); 1362 __ mov_metadata(mdo, md->constant_encoding()); 1363 if (mdo_offset_bias > 0) { 1364 __ mov_slow(tmp1, mdo_offset_bias); 1365 __ add(mdo, mdo, tmp1); 1366 } 1367 Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias); 1368 __ ldr(tmp1, data_addr); 1369 __ sub(tmp1, tmp1, DataLayout::counter_increment); 1370 __ str(tmp1, data_addr); 1371 __ b(*failure); 1372 } 1373 1374 // Sets `res` to true, if `cond` holds. On AArch64 also sets `res` to false if `cond` does not hold. 1375 static void set_instanceof_result(MacroAssembler* _masm, Register res, AsmCondition cond) { 1376 #ifdef AARCH64 1377 __ cset(res, cond); 1378 #else 1379 __ mov(res, 1, cond); 1380 #endif // AARCH64 1381 } 1382 1383 1384 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1385 // TODO: ARM - can be more effective with one more register 1386 switch (op->code()) { 1387 case lir_store_check: { 1388 CodeStub* stub = op->stub(); 1389 Register value = op->object()->as_register(); 1390 Register array = op->array()->as_register(); 1391 Register klass_RInfo = op->tmp1()->as_register(); 1392 Register k_RInfo = op->tmp2()->as_register(); 1393 assert_different_registers(klass_RInfo, k_RInfo, Rtemp); 1394 if (op->should_profile()) { 1395 assert_different_registers(value, klass_RInfo, k_RInfo, Rtemp); 1396 } 1397 1398 // check if it needs to be profiled 1399 ciMethodData* md; 1400 ciProfileData* data; 1401 int mdo_offset_bias = 0; 1402 Label profile_cast_success, profile_cast_failure, done; 1403 Label *success_target = op->should_profile() ? &profile_cast_success : &done; 1404 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry(); 1405 1406 if (op->should_profile()) { 1407 #ifndef AARCH64 1408 __ cmp(value, 0); 1409 #endif // !AARCH64 1410 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, value, k_RInfo, Rtemp, &done); 1411 } else { 1412 __ cbz(value, done); 1413 } 1414 assert_different_registers(k_RInfo, value); 1415 add_debug_info_for_null_check_here(op->info_for_exception()); 1416 __ load_klass(k_RInfo, array); 1417 __ load_klass(klass_RInfo, value); 1418 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1419 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1420 // check for immediate positive hit 1421 __ ldr(Rtemp, Address(klass_RInfo, Rtemp)); 1422 __ cmp(klass_RInfo, k_RInfo); 1423 __ cond_cmp(Rtemp, k_RInfo, ne); 1424 __ b(*success_target, eq); 1425 // check for immediate negative hit 1426 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1427 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset())); 1428 __ b(*failure_target, ne); 1429 // slow case 1430 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1431 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1432 __ cbz(R0, *failure_target); 1433 if (op->should_profile()) { 1434 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp; 1435 if (mdo == value) { 1436 mdo = k_RInfo; 1437 recv = klass_RInfo; 1438 } 1439 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, value, tmp1, 1440 &profile_cast_success, &profile_cast_failure, 1441 &done, stub->entry()); 1442 } 1443 __ bind(done); 1444 break; 1445 } 1446 1447 case lir_checkcast: { 1448 CodeStub* stub = op->stub(); 1449 Register obj = op->object()->as_register(); 1450 Register res = op->result_opr()->as_register(); 1451 Register klass_RInfo = op->tmp1()->as_register(); 1452 Register k_RInfo = op->tmp2()->as_register(); 1453 ciKlass* k = op->klass(); 1454 assert_different_registers(res, k_RInfo, klass_RInfo, Rtemp); 1455 1456 if (stub->is_simple_exception_stub()) { 1457 // TODO: ARM - Late binding is used to prevent confusion of register allocator 1458 assert(stub->is_exception_throw_stub(), "must be"); 1459 ((SimpleExceptionStub*)stub)->set_obj(op->result_opr()); 1460 } 1461 ciMethodData* md; 1462 ciProfileData* data; 1463 int mdo_offset_bias = 0; 1464 1465 Label done; 1466 1467 Label profile_cast_failure, profile_cast_success; 1468 Label *failure_target = op->should_profile() ? &profile_cast_failure : op->stub()->entry(); 1469 Label *success_target = op->should_profile() ? &profile_cast_success : &done; 1470 1471 #ifdef AARCH64 1472 move_regs(obj, res); 1473 if (op->should_profile()) { 1474 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done); 1475 } else { 1476 __ cbz(obj, done); 1477 } 1478 if (k->is_loaded()) { 1479 __ mov_metadata(k_RInfo, k->constant_encoding()); 1480 } else { 1481 if (res != obj) { 1482 op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res)); 1483 } 1484 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1485 } 1486 __ load_klass(klass_RInfo, res); 1487 1488 if (op->fast_check()) { 1489 __ cmp(klass_RInfo, k_RInfo); 1490 __ b(*failure_target, ne); 1491 } else if (k->is_loaded()) { 1492 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset())); 1493 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) { 1494 __ cmp(Rtemp, k_RInfo); 1495 __ b(*failure_target, ne); 1496 } else { 1497 __ cmp(klass_RInfo, k_RInfo); 1498 __ cond_cmp(Rtemp, k_RInfo, ne); 1499 __ b(*success_target, eq); 1500 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1501 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1502 __ cbz(R0, *failure_target); 1503 } 1504 } else { 1505 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1506 // check for immediate positive hit 1507 __ ldr(Rtemp, Address(klass_RInfo, Rtemp)); 1508 __ cmp(klass_RInfo, k_RInfo); 1509 __ cond_cmp(Rtemp, k_RInfo, ne); 1510 __ b(*success_target, eq); 1511 // check for immediate negative hit 1512 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1513 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset())); 1514 __ b(*failure_target, ne); 1515 // slow case 1516 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1517 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1518 __ cbz(R0, *failure_target); 1519 } 1520 1521 #else // AARCH64 1522 1523 __ movs(res, obj); 1524 if (op->should_profile()) { 1525 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done); 1526 } else { 1527 __ b(done, eq); 1528 } 1529 if (k->is_loaded()) { 1530 __ mov_metadata(k_RInfo, k->constant_encoding()); 1531 } else if (k_RInfo != obj) { 1532 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1533 __ movs(res, obj); 1534 } else { 1535 // Patching doesn't update "res" register after GC, so do patching first 1536 klass2reg_with_patching(Rtemp, op->info_for_patch()); 1537 __ movs(res, obj); 1538 __ mov(k_RInfo, Rtemp); 1539 } 1540 __ load_klass(klass_RInfo, res, ne); 1541 1542 if (op->fast_check()) { 1543 __ cmp(klass_RInfo, k_RInfo, ne); 1544 __ b(*failure_target, ne); 1545 } else if (k->is_loaded()) { 1546 __ b(*success_target, eq); 1547 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset())); 1548 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) { 1549 __ cmp(Rtemp, k_RInfo); 1550 __ b(*failure_target, ne); 1551 } else { 1552 __ cmp(klass_RInfo, k_RInfo); 1553 __ cmp(Rtemp, k_RInfo, ne); 1554 __ b(*success_target, eq); 1555 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1556 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1557 __ cbz(R0, *failure_target); 1558 } 1559 } else { 1560 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1561 __ b(*success_target, eq); 1562 // check for immediate positive hit 1563 __ ldr(Rtemp, Address(klass_RInfo, Rtemp)); 1564 __ cmp(klass_RInfo, k_RInfo); 1565 __ cmp(Rtemp, k_RInfo, ne); 1566 __ b(*success_target, eq); 1567 // check for immediate negative hit 1568 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1569 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset())); 1570 __ b(*failure_target, ne); 1571 // slow case 1572 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1573 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1574 __ cbz(R0, *failure_target); 1575 } 1576 #endif // AARCH64 1577 1578 if (op->should_profile()) { 1579 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp; 1580 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1, 1581 &profile_cast_success, &profile_cast_failure, 1582 &done, stub->entry()); 1583 } 1584 __ bind(done); 1585 break; 1586 } 1587 1588 case lir_instanceof: { 1589 Register obj = op->object()->as_register(); 1590 Register res = op->result_opr()->as_register(); 1591 Register klass_RInfo = op->tmp1()->as_register(); 1592 Register k_RInfo = op->tmp2()->as_register(); 1593 ciKlass* k = op->klass(); 1594 assert_different_registers(res, klass_RInfo, k_RInfo, Rtemp); 1595 1596 ciMethodData* md; 1597 ciProfileData* data; 1598 int mdo_offset_bias = 0; 1599 1600 Label done; 1601 1602 Label profile_cast_failure, profile_cast_success; 1603 Label *failure_target = op->should_profile() ? &profile_cast_failure : &done; 1604 Label *success_target = op->should_profile() ? &profile_cast_success : &done; 1605 1606 #ifdef AARCH64 1607 move_regs(obj, res); 1608 #else 1609 __ movs(res, obj); 1610 #endif // AARCH64 1611 1612 if (op->should_profile()) { 1613 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done); 1614 } else { 1615 #ifdef AARCH64 1616 __ cbz(obj, done); // If obj == NULL, res is false 1617 #else 1618 __ b(done, eq); 1619 #endif // AARCH64 1620 } 1621 1622 if (k->is_loaded()) { 1623 __ mov_metadata(k_RInfo, k->constant_encoding()); 1624 } else { 1625 op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res)); 1626 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1627 } 1628 __ load_klass(klass_RInfo, res); 1629 1630 #ifndef AARCH64 1631 if (!op->should_profile()) { 1632 __ mov(res, 0); 1633 } 1634 #endif // !AARCH64 1635 1636 if (op->fast_check()) { 1637 __ cmp(klass_RInfo, k_RInfo); 1638 if (!op->should_profile()) { 1639 set_instanceof_result(_masm, res, eq); 1640 } else { 1641 __ b(profile_cast_failure, ne); 1642 } 1643 } else if (k->is_loaded()) { 1644 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset())); 1645 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) { 1646 __ cmp(Rtemp, k_RInfo); 1647 if (!op->should_profile()) { 1648 set_instanceof_result(_masm, res, eq); 1649 } else { 1650 __ b(profile_cast_failure, ne); 1651 } 1652 } else { 1653 __ cmp(klass_RInfo, k_RInfo); 1654 __ cond_cmp(Rtemp, k_RInfo, ne); 1655 if (!op->should_profile()) { 1656 set_instanceof_result(_masm, res, eq); 1657 } 1658 __ b(*success_target, eq); 1659 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1660 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1661 if (!op->should_profile()) { 1662 move_regs(R0, res); 1663 } else { 1664 __ cbz(R0, *failure_target); 1665 } 1666 } 1667 } else { 1668 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1669 // check for immediate positive hit 1670 __ cmp(klass_RInfo, k_RInfo); 1671 if (!op->should_profile()) { 1672 #ifdef AARCH64 1673 // TODO-AARCH64 check if separate conditional branch is more efficient than ldr+cond_cmp 1674 __ ldr(res, Address(klass_RInfo, Rtemp)); 1675 #else 1676 __ ldr(res, Address(klass_RInfo, Rtemp), ne); 1677 #endif // AARCH64 1678 __ cond_cmp(res, k_RInfo, ne); 1679 set_instanceof_result(_masm, res, eq); 1680 } else { 1681 #ifdef AARCH64 1682 // TODO-AARCH64 check if separate conditional branch is more efficient than ldr+cond_cmp 1683 __ ldr(Rtemp, Address(klass_RInfo, Rtemp)); 1684 #else 1685 __ ldr(Rtemp, Address(klass_RInfo, Rtemp), ne); 1686 #endif // AARCH64 1687 __ cond_cmp(Rtemp, k_RInfo, ne); 1688 } 1689 __ b(*success_target, eq); 1690 // check for immediate negative hit 1691 if (op->should_profile()) { 1692 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1693 } 1694 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset())); 1695 if (!op->should_profile()) { 1696 #ifdef AARCH64 1697 __ mov(res, 0); 1698 #else 1699 __ mov(res, 0, ne); 1700 #endif // AARCH64 1701 } 1702 __ b(*failure_target, ne); 1703 // slow case 1704 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1705 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1706 if (!op->should_profile()) { 1707 move_regs(R0, res); 1708 } 1709 if (op->should_profile()) { 1710 __ cbz(R0, *failure_target); 1711 } 1712 } 1713 1714 if (op->should_profile()) { 1715 Label done_ok, done_failure; 1716 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp; 1717 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1, 1718 &profile_cast_success, &profile_cast_failure, 1719 &done_ok, &done_failure); 1720 __ bind(done_failure); 1721 __ mov(res, 0); 1722 __ b(done); 1723 __ bind(done_ok); 1724 __ mov(res, 1); 1725 } 1726 __ bind(done); 1727 break; 1728 } 1729 default: 1730 ShouldNotReachHere(); 1731 } 1732 } 1733 1734 1735 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1736 // if (*addr == cmpval) { 1737 // *addr = newval; 1738 // dest = 1; 1739 // } else { 1740 // dest = 0; 1741 // } 1742 #ifdef AARCH64 1743 Label retry, done; 1744 Register addr = op->addr()->as_pointer_register(); 1745 Register cmpval = op->cmp_value()->as_pointer_register(); 1746 Register newval = op->new_value()->as_pointer_register(); 1747 Register dest = op->result_opr()->as_pointer_register(); 1748 assert_different_registers(dest, addr, cmpval, newval, Rtemp); 1749 1750 if (UseCompressedOops && op->code() == lir_cas_obj) { 1751 Register tmp1 = op->tmp1()->as_pointer_register(); 1752 Register tmp2 = op->tmp2()->as_pointer_register(); 1753 assert_different_registers(dest, addr, cmpval, newval, tmp1, tmp2, Rtemp); 1754 __ encode_heap_oop(tmp1, cmpval); cmpval = tmp1; 1755 __ encode_heap_oop(tmp2, newval); newval = tmp2; 1756 } 1757 1758 __ mov(dest, ZR); 1759 __ bind(retry); 1760 if (((op->code() == lir_cas_obj) && !UseCompressedOops) || op->code() == lir_cas_long) { 1761 __ ldaxr(Rtemp, addr); 1762 __ cmp(Rtemp, cmpval); 1763 __ b(done, ne); 1764 __ stlxr(Rtemp, newval, addr); 1765 } else if (((op->code() == lir_cas_obj) && UseCompressedOops) || op->code() == lir_cas_int) { 1766 __ ldaxr_w(Rtemp, addr); 1767 __ cmp_w(Rtemp, cmpval); 1768 __ b(done, ne); 1769 __ stlxr_w(Rtemp, newval, addr); 1770 } else { 1771 ShouldNotReachHere(); 1772 } 1773 __ cbnz_w(Rtemp, retry); 1774 __ mov(dest, 1); 1775 __ bind(done); 1776 #else 1777 // FIXME: membar_release 1778 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp); 1779 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) { 1780 Register addr = op->addr()->as_register(); 1781 Register cmpval = op->cmp_value()->as_register(); 1782 Register newval = op->new_value()->as_register(); 1783 Register dest = op->result_opr()->as_register(); 1784 assert_different_registers(dest, addr, cmpval, newval, Rtemp); 1785 1786 __ atomic_cas_bool(cmpval, newval, addr, 0, Rtemp); // Rtemp free by default at C1 LIR layer 1787 __ mov(dest, 1, eq); 1788 __ mov(dest, 0, ne); 1789 } else if (op->code() == lir_cas_long) { 1790 assert(VM_Version::supports_cx8(), "wrong machine"); 1791 Register addr = op->addr()->as_pointer_register(); 1792 Register cmp_value_lo = op->cmp_value()->as_register_lo(); 1793 Register cmp_value_hi = op->cmp_value()->as_register_hi(); 1794 Register new_value_lo = op->new_value()->as_register_lo(); 1795 Register new_value_hi = op->new_value()->as_register_hi(); 1796 Register dest = op->result_opr()->as_register(); 1797 Register tmp_lo = op->tmp1()->as_register_lo(); 1798 Register tmp_hi = op->tmp1()->as_register_hi(); 1799 1800 assert_different_registers(tmp_lo, tmp_hi, cmp_value_lo, cmp_value_hi, dest, new_value_lo, new_value_hi, addr); 1801 assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair"); 1802 assert(new_value_hi->encoding() == new_value_lo->encoding() + 1, "non aligned register pair"); 1803 assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair"); 1804 assert((new_value_lo->encoding() & 0x1) == 0, "misaligned register pair"); 1805 __ atomic_cas64(tmp_lo, tmp_hi, dest, cmp_value_lo, cmp_value_hi, 1806 new_value_lo, new_value_hi, addr, 0); 1807 } else { 1808 Unimplemented(); 1809 } 1810 #endif // AARCH64 1811 // FIXME: is full membar really needed instead of just membar_acquire? 1812 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp); 1813 } 1814 1815 1816 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) { 1817 AsmCondition acond = al; 1818 AsmCondition ncond = nv; 1819 if (opr1 != opr2) { 1820 switch (condition) { 1821 case lir_cond_equal: acond = eq; ncond = ne; break; 1822 case lir_cond_notEqual: acond = ne; ncond = eq; break; 1823 case lir_cond_less: acond = lt; ncond = ge; break; 1824 case lir_cond_lessEqual: acond = le; ncond = gt; break; 1825 case lir_cond_greaterEqual: acond = ge; ncond = lt; break; 1826 case lir_cond_greater: acond = gt; ncond = le; break; 1827 case lir_cond_aboveEqual: acond = hs; ncond = lo; break; 1828 case lir_cond_belowEqual: acond = ls; ncond = hi; break; 1829 default: ShouldNotReachHere(); 1830 } 1831 } 1832 1833 #ifdef AARCH64 1834 1835 // TODO-AARCH64 implement it more efficiently 1836 1837 if (opr1->is_register()) { 1838 reg2reg(opr1, result); 1839 } else if (opr1->is_stack()) { 1840 stack2reg(opr1, result, result->type()); 1841 } else if (opr1->is_constant()) { 1842 const2reg(opr1, result, lir_patch_none, NULL); 1843 } else { 1844 ShouldNotReachHere(); 1845 } 1846 1847 Label skip; 1848 __ b(skip, acond); 1849 1850 if (opr2->is_register()) { 1851 reg2reg(opr2, result); 1852 } else if (opr2->is_stack()) { 1853 stack2reg(opr2, result, result->type()); 1854 } else if (opr2->is_constant()) { 1855 const2reg(opr2, result, lir_patch_none, NULL); 1856 } else { 1857 ShouldNotReachHere(); 1858 } 1859 1860 __ bind(skip); 1861 1862 #else 1863 for (;;) { // two iterations only 1864 if (opr1 == result) { 1865 // do nothing 1866 } else if (opr1->is_single_cpu()) { 1867 __ mov(result->as_register(), opr1->as_register(), acond); 1868 } else if (opr1->is_double_cpu()) { 1869 __ long_move(result->as_register_lo(), result->as_register_hi(), 1870 opr1->as_register_lo(), opr1->as_register_hi(), acond); 1871 } else if (opr1->is_single_stack()) { 1872 __ ldr(result->as_register(), frame_map()->address_for_slot(opr1->single_stack_ix()), acond); 1873 } else if (opr1->is_double_stack()) { 1874 __ ldr(result->as_register_lo(), 1875 frame_map()->address_for_slot(opr1->double_stack_ix(), lo_word_offset_in_bytes), acond); 1876 __ ldr(result->as_register_hi(), 1877 frame_map()->address_for_slot(opr1->double_stack_ix(), hi_word_offset_in_bytes), acond); 1878 } else if (opr1->is_illegal()) { 1879 // do nothing: this part of the cmove has been optimized away in the peephole optimizer 1880 } else { 1881 assert(opr1->is_constant(), "must be"); 1882 LIR_Const* c = opr1->as_constant_ptr(); 1883 1884 switch (c->type()) { 1885 case T_INT: 1886 __ mov_slow(result->as_register(), c->as_jint(), acond); 1887 break; 1888 case T_LONG: 1889 __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond); 1890 __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond); 1891 break; 1892 case T_OBJECT: 1893 __ mov_oop(result->as_register(), c->as_jobject(), 0, acond); 1894 break; 1895 case T_FLOAT: 1896 #ifdef __SOFTFP__ 1897 // not generated now. 1898 __ mov_slow(result->as_register(), c->as_jint(), acond); 1899 #else 1900 __ mov_float(result->as_float_reg(), c->as_jfloat(), acond); 1901 #endif // __SOFTFP__ 1902 break; 1903 case T_DOUBLE: 1904 #ifdef __SOFTFP__ 1905 // not generated now. 1906 __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond); 1907 __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond); 1908 #else 1909 __ mov_double(result->as_double_reg(), c->as_jdouble(), acond); 1910 #endif // __SOFTFP__ 1911 break; 1912 default: 1913 ShouldNotReachHere(); 1914 } 1915 } 1916 1917 // Negate the condition and repeat the algorithm with the second operand 1918 if (opr1 == opr2) { break; } 1919 opr1 = opr2; 1920 acond = ncond; 1921 } 1922 #endif // AARCH64 1923 } 1924 1925 #if defined(AARCH64) || defined(ASSERT) 1926 static int reg_size(LIR_Opr op) { 1927 switch (op->type()) { 1928 case T_FLOAT: 1929 case T_INT: return BytesPerInt; 1930 case T_LONG: 1931 case T_DOUBLE: return BytesPerLong; 1932 case T_OBJECT: 1933 case T_ARRAY: 1934 case T_METADATA: return BytesPerWord; 1935 case T_ADDRESS: 1936 case T_ILLEGAL: // fall through 1937 default: ShouldNotReachHere(); return -1; 1938 } 1939 } 1940 #endif 1941 1942 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 1943 assert(info == NULL, "unused on this code path"); 1944 assert(dest->is_register(), "wrong items state"); 1945 1946 if (right->is_address()) { 1947 // special case for adding shifted/extended register 1948 const Register res = dest->as_pointer_register(); 1949 const Register lreg = left->as_pointer_register(); 1950 const LIR_Address* addr = right->as_address_ptr(); 1951 1952 assert(addr->base()->as_pointer_register() == lreg && addr->index()->is_register() && addr->disp() == 0, "must be"); 1953 1954 int scale = addr->scale(); 1955 AsmShift shift = lsl; 1956 1957 #ifdef AARCH64 1958 bool is_index_extended = reg_size(addr->base()) > reg_size(addr->index()); 1959 if (scale < 0) { 1960 scale = -scale; 1961 shift = lsr; 1962 } 1963 assert(shift == lsl || !is_index_extended, "could not have extend and right shift in one operand"); 1964 assert(0 <= scale && scale <= 63, "scale is too large"); 1965 1966 if (is_index_extended) { 1967 assert(scale <= 4, "scale is too large for add with extended register"); 1968 assert(addr->index()->is_single_cpu(), "should be"); 1969 assert(addr->index()->type() == T_INT, "should be"); 1970 assert(dest->is_double_cpu(), "should be"); 1971 assert(code == lir_add, "special case of add with extended register"); 1972 1973 __ add(res, lreg, addr->index()->as_register(), ex_sxtw, scale); 1974 return; 1975 } else if (reg_size(dest) == BytesPerInt) { 1976 assert(reg_size(addr->base()) == reg_size(addr->index()), "should be"); 1977 assert(reg_size(addr->base()) == reg_size(dest), "should be"); 1978 1979 AsmOperand operand(addr->index()->as_pointer_register(), shift, scale); 1980 switch (code) { 1981 case lir_add: __ add_32(res, lreg, operand); break; 1982 case lir_sub: __ sub_32(res, lreg, operand); break; 1983 default: ShouldNotReachHere(); 1984 } 1985 return; 1986 } 1987 #endif // AARCH64 1988 1989 assert(reg_size(addr->base()) == reg_size(addr->index()), "should be"); 1990 assert(reg_size(addr->base()) == reg_size(dest), "should be"); 1991 assert(reg_size(dest) == wordSize, "should be"); 1992 1993 AsmOperand operand(addr->index()->as_pointer_register(), shift, scale); 1994 switch (code) { 1995 case lir_add: __ add(res, lreg, operand); break; 1996 case lir_sub: __ sub(res, lreg, operand); break; 1997 default: ShouldNotReachHere(); 1998 } 1999 2000 #ifndef AARCH64 2001 } else if (left->is_address()) { 2002 assert(code == lir_sub && right->is_single_cpu(), "special case used by strength_reduce_multiply()"); 2003 const LIR_Address* addr = left->as_address_ptr(); 2004 const Register res = dest->as_register(); 2005 const Register rreg = right->as_register(); 2006 assert(addr->base()->as_register() == rreg && addr->index()->is_register() && addr->disp() == 0, "must be"); 2007 __ rsb(res, rreg, AsmOperand(addr->index()->as_register(), lsl, addr->scale())); 2008 #endif // !AARCH64 2009 2010 } else if (dest->is_single_cpu()) { 2011 assert(left->is_single_cpu(), "unexpected left operand"); 2012 #ifdef AARCH64 2013 assert(dest->type() == T_INT, "unexpected dest type"); 2014 assert(left->type() == T_INT, "unexpected left type"); 2015 assert(right->type() == T_INT, "unexpected right type"); 2016 #endif // AARCH64 2017 2018 const Register res = dest->as_register(); 2019 const Register lreg = left->as_register(); 2020 2021 if (right->is_single_cpu()) { 2022 const Register rreg = right->as_register(); 2023 switch (code) { 2024 case lir_add: __ add_32(res, lreg, rreg); break; 2025 case lir_sub: __ sub_32(res, lreg, rreg); break; 2026 case lir_mul: __ mul_32(res, lreg, rreg); break; 2027 default: ShouldNotReachHere(); 2028 } 2029 } else { 2030 assert(right->is_constant(), "must be"); 2031 const jint c = right->as_constant_ptr()->as_jint(); 2032 if (!Assembler::is_arith_imm_in_range(c)) { 2033 BAILOUT("illegal arithmetic operand"); 2034 } 2035 switch (code) { 2036 case lir_add: __ add_32(res, lreg, c); break; 2037 case lir_sub: __ sub_32(res, lreg, c); break; 2038 default: ShouldNotReachHere(); 2039 } 2040 } 2041 2042 } else if (dest->is_double_cpu()) { 2043 #ifdef AARCH64 2044 assert(left->is_double_cpu() || 2045 (left->is_single_cpu() && ((left->type() == T_OBJECT) || (left->type() == T_ARRAY) || (left->type() == T_ADDRESS))), 2046 "unexpected left operand"); 2047 2048 const Register res = dest->as_register_lo(); 2049 const Register lreg = left->as_pointer_register(); 2050 2051 if (right->is_constant()) { 2052 assert(right->type() == T_LONG, "unexpected right type"); 2053 assert((right->as_constant_ptr()->as_jlong() >> 24) == 0, "out of range"); 2054 jint imm = (jint)right->as_constant_ptr()->as_jlong(); 2055 switch (code) { 2056 case lir_add: __ add(res, lreg, imm); break; 2057 case lir_sub: __ sub(res, lreg, imm); break; 2058 default: ShouldNotReachHere(); 2059 } 2060 } else { 2061 assert(right->is_double_cpu() || 2062 (right->is_single_cpu() && ((right->type() == T_OBJECT) || (right->type() == T_ARRAY) || (right->type() == T_ADDRESS))), 2063 "unexpected right operand"); 2064 const Register rreg = right->as_pointer_register(); 2065 switch (code) { 2066 case lir_add: __ add(res, lreg, rreg); break; 2067 case lir_sub: __ sub(res, lreg, rreg); break; 2068 case lir_mul: __ mul(res, lreg, rreg); break; 2069 default: ShouldNotReachHere(); 2070 } 2071 } 2072 #else // AARCH64 2073 Register res_lo = dest->as_register_lo(); 2074 Register res_hi = dest->as_register_hi(); 2075 Register lreg_lo = left->as_register_lo(); 2076 Register lreg_hi = left->as_register_hi(); 2077 if (right->is_double_cpu()) { 2078 Register rreg_lo = right->as_register_lo(); 2079 Register rreg_hi = right->as_register_hi(); 2080 if (res_lo == lreg_hi || res_lo == rreg_hi) { 2081 res_lo = Rtemp; 2082 } 2083 switch (code) { 2084 case lir_add: 2085 __ adds(res_lo, lreg_lo, rreg_lo); 2086 __ adc(res_hi, lreg_hi, rreg_hi); 2087 break; 2088 case lir_sub: 2089 __ subs(res_lo, lreg_lo, rreg_lo); 2090 __ sbc(res_hi, lreg_hi, rreg_hi); 2091 break; 2092 default: 2093 ShouldNotReachHere(); 2094 } 2095 } else { 2096 assert(right->is_constant(), "must be"); 2097 assert((right->as_constant_ptr()->as_jlong() >> 32) == 0, "out of range"); 2098 const jint c = (jint) right->as_constant_ptr()->as_jlong(); 2099 if (res_lo == lreg_hi) { 2100 res_lo = Rtemp; 2101 } 2102 switch (code) { 2103 case lir_add: 2104 __ adds(res_lo, lreg_lo, c); 2105 __ adc(res_hi, lreg_hi, 0); 2106 break; 2107 case lir_sub: 2108 __ subs(res_lo, lreg_lo, c); 2109 __ sbc(res_hi, lreg_hi, 0); 2110 break; 2111 default: 2112 ShouldNotReachHere(); 2113 } 2114 } 2115 move_regs(res_lo, dest->as_register_lo()); 2116 #endif // AARCH64 2117 2118 } else if (dest->is_single_fpu()) { 2119 assert(left->is_single_fpu(), "must be"); 2120 assert(right->is_single_fpu(), "must be"); 2121 const FloatRegister res = dest->as_float_reg(); 2122 const FloatRegister lreg = left->as_float_reg(); 2123 const FloatRegister rreg = right->as_float_reg(); 2124 switch (code) { 2125 case lir_add: __ add_float(res, lreg, rreg); break; 2126 case lir_sub: __ sub_float(res, lreg, rreg); break; 2127 case lir_mul_strictfp: // fall through 2128 case lir_mul: __ mul_float(res, lreg, rreg); break; 2129 case lir_div_strictfp: // fall through 2130 case lir_div: __ div_float(res, lreg, rreg); break; 2131 default: ShouldNotReachHere(); 2132 } 2133 } else if (dest->is_double_fpu()) { 2134 assert(left->is_double_fpu(), "must be"); 2135 assert(right->is_double_fpu(), "must be"); 2136 const FloatRegister res = dest->as_double_reg(); 2137 const FloatRegister lreg = left->as_double_reg(); 2138 const FloatRegister rreg = right->as_double_reg(); 2139 switch (code) { 2140 case lir_add: __ add_double(res, lreg, rreg); break; 2141 case lir_sub: __ sub_double(res, lreg, rreg); break; 2142 case lir_mul_strictfp: // fall through 2143 case lir_mul: __ mul_double(res, lreg, rreg); break; 2144 case lir_div_strictfp: // fall through 2145 case lir_div: __ div_double(res, lreg, rreg); break; 2146 default: ShouldNotReachHere(); 2147 } 2148 } else { 2149 ShouldNotReachHere(); 2150 } 2151 } 2152 2153 2154 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { 2155 switch (code) { 2156 case lir_abs: 2157 __ abs_double(dest->as_double_reg(), value->as_double_reg()); 2158 break; 2159 case lir_sqrt: 2160 __ sqrt_double(dest->as_double_reg(), value->as_double_reg()); 2161 break; 2162 default: 2163 ShouldNotReachHere(); 2164 } 2165 } 2166 2167 2168 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) { 2169 assert(dest->is_register(), "wrong items state"); 2170 assert(left->is_register(), "wrong items state"); 2171 2172 if (dest->is_single_cpu()) { 2173 #ifdef AARCH64 2174 assert (dest->type() == T_INT, "unexpected result type"); 2175 assert (left->type() == T_INT, "unexpected left type"); 2176 assert (right->type() == T_INT, "unexpected right type"); 2177 #endif // AARCH64 2178 2179 const Register res = dest->as_register(); 2180 const Register lreg = left->as_register(); 2181 2182 if (right->is_single_cpu()) { 2183 const Register rreg = right->as_register(); 2184 switch (code) { 2185 case lir_logic_and: __ and_32(res, lreg, rreg); break; 2186 case lir_logic_or: __ orr_32(res, lreg, rreg); break; 2187 case lir_logic_xor: __ eor_32(res, lreg, rreg); break; 2188 default: ShouldNotReachHere(); 2189 } 2190 } else { 2191 assert(right->is_constant(), "must be"); 2192 const uint c = (uint)right->as_constant_ptr()->as_jint(); 2193 switch (code) { 2194 case lir_logic_and: __ and_32(res, lreg, c); break; 2195 case lir_logic_or: __ orr_32(res, lreg, c); break; 2196 case lir_logic_xor: __ eor_32(res, lreg, c); break; 2197 default: ShouldNotReachHere(); 2198 } 2199 } 2200 } else { 2201 assert(dest->is_double_cpu(), "should be"); 2202 Register res_lo = dest->as_register_lo(); 2203 2204 #ifdef AARCH64 2205 assert ((left->is_single_cpu() && left->is_oop_register()) || left->is_double_cpu(), "should be"); 2206 const Register lreg_lo = left->as_pointer_register(); 2207 #else 2208 assert (dest->type() == T_LONG, "unexpected result type"); 2209 assert (left->type() == T_LONG, "unexpected left type"); 2210 assert (right->type() == T_LONG, "unexpected right type"); 2211 2212 const Register res_hi = dest->as_register_hi(); 2213 const Register lreg_lo = left->as_register_lo(); 2214 const Register lreg_hi = left->as_register_hi(); 2215 #endif // AARCH64 2216 2217 if (right->is_register()) { 2218 #ifdef AARCH64 2219 assert ((right->is_single_cpu() && right->is_oop_register()) || right->is_double_cpu(), "should be"); 2220 const Register rreg_lo = right->as_pointer_register(); 2221 switch (code) { 2222 case lir_logic_and: __ andr(res_lo, lreg_lo, rreg_lo); break; 2223 case lir_logic_or: __ orr (res_lo, lreg_lo, rreg_lo); break; 2224 case lir_logic_xor: __ eor (res_lo, lreg_lo, rreg_lo); break; 2225 default: ShouldNotReachHere(); 2226 } 2227 #else 2228 const Register rreg_lo = right->as_register_lo(); 2229 const Register rreg_hi = right->as_register_hi(); 2230 if (res_lo == lreg_hi || res_lo == rreg_hi) { 2231 res_lo = Rtemp; // Temp register helps to avoid overlap between result and input 2232 } 2233 switch (code) { 2234 case lir_logic_and: 2235 __ andr(res_lo, lreg_lo, rreg_lo); 2236 __ andr(res_hi, lreg_hi, rreg_hi); 2237 break; 2238 case lir_logic_or: 2239 __ orr(res_lo, lreg_lo, rreg_lo); 2240 __ orr(res_hi, lreg_hi, rreg_hi); 2241 break; 2242 case lir_logic_xor: 2243 __ eor(res_lo, lreg_lo, rreg_lo); 2244 __ eor(res_hi, lreg_hi, rreg_hi); 2245 break; 2246 default: 2247 ShouldNotReachHere(); 2248 } 2249 move_regs(res_lo, dest->as_register_lo()); 2250 #endif // AARCH64 2251 } else { 2252 assert(right->is_constant(), "must be"); 2253 #ifdef AARCH64 2254 const julong c = (julong)right->as_constant_ptr()->as_jlong(); 2255 Assembler::LogicalImmediate imm(c, false); 2256 if (imm.is_encoded()) { 2257 switch (code) { 2258 case lir_logic_and: __ andr(res_lo, lreg_lo, imm); break; 2259 case lir_logic_or: __ orr (res_lo, lreg_lo, imm); break; 2260 case lir_logic_xor: __ eor (res_lo, lreg_lo, imm); break; 2261 default: ShouldNotReachHere(); 2262 } 2263 } else { 2264 BAILOUT("64 bit constant cannot be inlined"); 2265 } 2266 #else 2267 const jint c_lo = (jint) right->as_constant_ptr()->as_jlong(); 2268 const jint c_hi = (jint) (right->as_constant_ptr()->as_jlong() >> 32); 2269 // Case for logic_or from do_ClassIDIntrinsic() 2270 if (c_hi == 0 && AsmOperand::is_rotated_imm(c_lo)) { 2271 switch (code) { 2272 case lir_logic_and: 2273 __ andr(res_lo, lreg_lo, c_lo); 2274 __ mov(res_hi, 0); 2275 break; 2276 case lir_logic_or: 2277 __ orr(res_lo, lreg_lo, c_lo); 2278 break; 2279 case lir_logic_xor: 2280 __ eor(res_lo, lreg_lo, c_lo); 2281 break; 2282 default: 2283 ShouldNotReachHere(); 2284 } 2285 } else if (code == lir_logic_and && 2286 c_hi == -1 && 2287 (AsmOperand::is_rotated_imm(c_lo) || 2288 AsmOperand::is_rotated_imm(~c_lo))) { 2289 // Another case which handles logic_and from do_ClassIDIntrinsic() 2290 if (AsmOperand::is_rotated_imm(c_lo)) { 2291 __ andr(res_lo, lreg_lo, c_lo); 2292 } else { 2293 __ bic(res_lo, lreg_lo, ~c_lo); 2294 } 2295 if (res_hi != lreg_hi) { 2296 __ mov(res_hi, lreg_hi); 2297 } 2298 } else { 2299 BAILOUT("64 bit constant cannot be inlined"); 2300 } 2301 #endif // AARCH64 2302 } 2303 } 2304 } 2305 2306 2307 #ifdef AARCH64 2308 2309 void LIR_Assembler::long_compare_helper(LIR_Opr opr1, LIR_Opr opr2) { 2310 assert(opr1->is_double_cpu(), "should be"); 2311 Register x = opr1->as_register_lo(); 2312 2313 if (opr2->is_double_cpu()) { 2314 Register y = opr2->as_register_lo(); 2315 __ cmp(x, y); 2316 2317 } else { 2318 assert(opr2->is_constant(), "should be"); 2319 assert(opr2->as_constant_ptr()->type() == T_LONG, "long constant expected"); 2320 jlong c = opr2->as_jlong(); 2321 assert(((c >> 31) == 0) || ((c >> 31) == -1), "immediate is out of range"); 2322 if (c >= 0) { 2323 __ cmp(x, (jint)c); 2324 } else { 2325 __ cmn(x, (jint)(-c)); 2326 } 2327 } 2328 } 2329 2330 #endif // AARCH64 2331 2332 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 2333 if (opr1->is_single_cpu()) { 2334 if (opr2->is_constant()) { 2335 switch (opr2->as_constant_ptr()->type()) { 2336 case T_INT: { 2337 const jint c = opr2->as_constant_ptr()->as_jint(); 2338 if (Assembler::is_arith_imm_in_range(c)) { 2339 __ cmp_32(opr1->as_register(), c); 2340 } else if (Assembler::is_arith_imm_in_range(-c)) { 2341 __ cmn_32(opr1->as_register(), -c); 2342 } else { 2343 // This can happen when compiling lookupswitch 2344 __ mov_slow(Rtemp, c); 2345 __ cmp_32(opr1->as_register(), Rtemp); 2346 } 2347 break; 2348 } 2349 case T_OBJECT: 2350 assert(opr2->as_constant_ptr()->as_jobject() == NULL, "cannot handle otherwise"); 2351 __ cmp(opr1->as_register(), 0); 2352 break; 2353 default: 2354 ShouldNotReachHere(); 2355 } 2356 } else if (opr2->is_single_cpu()) { 2357 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_METADATA || opr1->type() == T_ADDRESS) { 2358 assert(opr2->type() == T_OBJECT || opr2->type() == T_ARRAY || opr2->type() == T_METADATA || opr2->type() == T_ADDRESS, "incompatibe type"); 2359 __ cmp(opr1->as_register(), opr2->as_register()); 2360 } else { 2361 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY && opr2->type() != T_METADATA && opr2->type() != T_ADDRESS, "incompatibe type"); 2362 __ cmp_32(opr1->as_register(), opr2->as_register()); 2363 } 2364 } else { 2365 ShouldNotReachHere(); 2366 } 2367 } else if (opr1->is_double_cpu()) { 2368 #ifdef AARCH64 2369 long_compare_helper(opr1, opr2); 2370 #else 2371 Register xlo = opr1->as_register_lo(); 2372 Register xhi = opr1->as_register_hi(); 2373 if (opr2->is_constant() && opr2->as_jlong() == 0) { 2374 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "cannot handle otherwise"); 2375 __ orrs(Rtemp, xlo, xhi); 2376 } else if (opr2->is_register()) { 2377 Register ylo = opr2->as_register_lo(); 2378 Register yhi = opr2->as_register_hi(); 2379 if (condition == lir_cond_equal || condition == lir_cond_notEqual) { 2380 __ teq(xhi, yhi); 2381 __ teq(xlo, ylo, eq); 2382 } else { 2383 __ subs(xlo, xlo, ylo); 2384 __ sbcs(xhi, xhi, yhi); 2385 } 2386 } else { 2387 ShouldNotReachHere(); 2388 } 2389 #endif // AARCH64 2390 } else if (opr1->is_single_fpu()) { 2391 if (opr2->is_constant()) { 2392 assert(opr2->as_jfloat() == 0.0f, "cannot handle otherwise"); 2393 __ cmp_zero_float(opr1->as_float_reg()); 2394 } else { 2395 __ cmp_float(opr1->as_float_reg(), opr2->as_float_reg()); 2396 } 2397 } else if (opr1->is_double_fpu()) { 2398 if (opr2->is_constant()) { 2399 assert(opr2->as_jdouble() == 0.0, "cannot handle otherwise"); 2400 __ cmp_zero_double(opr1->as_double_reg()); 2401 } else { 2402 __ cmp_double(opr1->as_double_reg(), opr2->as_double_reg()); 2403 } 2404 } else { 2405 ShouldNotReachHere(); 2406 } 2407 } 2408 2409 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) { 2410 const Register res = dst->as_register(); 2411 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 2412 comp_op(lir_cond_unknown, left, right, op); 2413 #ifdef AARCH64 2414 if (code == lir_ucmp_fd2i) { // unordered is less 2415 __ cset(res, gt); // 1 if '>', else 0 2416 __ csinv(res, res, ZR, ge); // previous value if '>=', else -1 2417 } else { 2418 __ cset(res, hi); // 1 if '>' or unordered, else 0 2419 __ csinv(res, res, ZR, pl); // previous value if '>=' or unordered, else -1 2420 } 2421 #else 2422 __ fmstat(); 2423 if (code == lir_ucmp_fd2i) { // unordered is less 2424 __ mvn(res, 0, lt); 2425 __ mov(res, 1, ge); 2426 } else { // unordered is greater 2427 __ mov(res, 1, cs); 2428 __ mvn(res, 0, cc); 2429 } 2430 __ mov(res, 0, eq); 2431 #endif // AARCH64 2432 2433 } else { 2434 assert(code == lir_cmp_l2i, "must be"); 2435 2436 #ifdef AARCH64 2437 long_compare_helper(left, right); 2438 2439 __ cset(res, gt); // 1 if '>', else 0 2440 __ csinv(res, res, ZR, ge); // previous value if '>=', else -1 2441 #else 2442 Label done; 2443 const Register xlo = left->as_register_lo(); 2444 const Register xhi = left->as_register_hi(); 2445 const Register ylo = right->as_register_lo(); 2446 const Register yhi = right->as_register_hi(); 2447 __ cmp(xhi, yhi); 2448 __ mov(res, 1, gt); 2449 __ mvn(res, 0, lt); 2450 __ b(done, ne); 2451 __ subs(res, xlo, ylo); 2452 __ mov(res, 1, hi); 2453 __ mvn(res, 0, lo); 2454 __ bind(done); 2455 #endif // AARCH64 2456 } 2457 } 2458 2459 2460 void LIR_Assembler::align_call(LIR_Code code) { 2461 // Not needed 2462 } 2463 2464 2465 void LIR_Assembler::call(LIR_OpJavaCall *op, relocInfo::relocType rtype) { 2466 int ret_addr_offset = __ patchable_call(op->addr(), rtype); 2467 assert(ret_addr_offset == __ offset(), "embedded return address not allowed"); 2468 add_call_info_here(op->info()); 2469 } 2470 2471 2472 void LIR_Assembler::ic_call(LIR_OpJavaCall *op) { 2473 bool near_range = __ cache_fully_reachable(); 2474 address oop_address = pc(); 2475 2476 bool use_movw = AARCH64_ONLY(false) NOT_AARCH64(VM_Version::supports_movw()); 2477 2478 // Ricklass may contain something that is not a metadata pointer so 2479 // mov_metadata can't be used 2480 InlinedAddress value((address)Universe::non_oop_word()); 2481 InlinedAddress addr(op->addr()); 2482 if (use_movw) { 2483 #ifdef AARCH64 2484 ShouldNotReachHere(); 2485 #else 2486 __ movw(Ricklass, ((unsigned int)Universe::non_oop_word()) & 0xffff); 2487 __ movt(Ricklass, ((unsigned int)Universe::non_oop_word()) >> 16); 2488 #endif // AARCH64 2489 } else { 2490 // No movw/movt, must be load a pc relative value but no 2491 // relocation so no metadata table to load from. 2492 // Use a b instruction rather than a bl, inline constant after the 2493 // branch, use a PC relative ldr to load the constant, arrange for 2494 // the call to return after the constant(s). 2495 __ ldr_literal(Ricklass, value); 2496 } 2497 __ relocate(virtual_call_Relocation::spec(oop_address)); 2498 if (near_range && use_movw) { 2499 __ bl(op->addr()); 2500 } else { 2501 Label call_return; 2502 __ adr(LR, call_return); 2503 if (near_range) { 2504 __ b(op->addr()); 2505 } else { 2506 __ indirect_jump(addr, Rtemp); 2507 __ bind_literal(addr); 2508 } 2509 if (!use_movw) { 2510 __ bind_literal(value); 2511 } 2512 __ bind(call_return); 2513 } 2514 add_call_info(code_offset(), op->info()); 2515 } 2516 2517 2518 /* Currently, vtable-dispatch is only enabled for sparc platforms */ 2519 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) { 2520 ShouldNotReachHere(); 2521 } 2522 2523 void LIR_Assembler::emit_static_call_stub() { 2524 address call_pc = __ pc(); 2525 address stub = __ start_a_stub(call_stub_size()); 2526 if (stub == NULL) { 2527 BAILOUT("static call stub overflow"); 2528 } 2529 2530 DEBUG_ONLY(int offset = code_offset();) 2531 2532 InlinedMetadata metadata_literal(NULL); 2533 __ relocate(static_stub_Relocation::spec(call_pc)); 2534 // If not a single instruction, NativeMovConstReg::next_instruction_address() 2535 // must jump over the whole following ldr_literal. 2536 // (See CompiledStaticCall::set_to_interpreted()) 2537 #ifdef ASSERT 2538 address ldr_site = __ pc(); 2539 #endif 2540 __ ldr_literal(Rmethod, metadata_literal); 2541 assert(nativeMovConstReg_at(ldr_site)->next_instruction_address() == __ pc(), "Fix ldr_literal or its parsing"); 2542 bool near_range = __ cache_fully_reachable(); 2543 InlinedAddress dest((address)-1); 2544 if (near_range) { 2545 address branch_site = __ pc(); 2546 __ b(branch_site); // b to self maps to special NativeJump -1 destination 2547 } else { 2548 __ indirect_jump(dest, Rtemp); 2549 } 2550 __ bind_literal(metadata_literal); // includes spec_for_immediate reloc 2551 if (!near_range) { 2552 __ bind_literal(dest); // special NativeJump -1 destination 2553 } 2554 2555 assert(code_offset() - offset <= call_stub_size(), "overflow"); 2556 __ end_a_stub(); 2557 } 2558 2559 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 2560 assert(exceptionOop->as_register() == Rexception_obj, "must match"); 2561 assert(exceptionPC->as_register() == Rexception_pc, "must match"); 2562 info->add_register_oop(exceptionOop); 2563 2564 Runtime1::StubID handle_id = compilation()->has_fpu_code() ? 2565 Runtime1::handle_exception_id : 2566 Runtime1::handle_exception_nofpu_id; 2567 Label return_address; 2568 __ adr(Rexception_pc, return_address); 2569 __ call(Runtime1::entry_for(handle_id), relocInfo::runtime_call_type); 2570 __ bind(return_address); 2571 add_call_info_here(info); // for exception handler 2572 } 2573 2574 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 2575 assert(exceptionOop->as_register() == Rexception_obj, "must match"); 2576 __ b(_unwind_handler_entry); 2577 } 2578 2579 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 2580 #ifdef AARCH64 2581 if (dest->is_single_cpu()) { 2582 Register res = dest->as_register(); 2583 Register x = left->as_register(); 2584 Register y = count->as_register(); 2585 assert (dest->type() == T_INT, "unexpected result type"); 2586 assert (left->type() == T_INT, "unexpected left type"); 2587 2588 switch (code) { 2589 case lir_shl: __ lslv_w(res, x, y); break; 2590 case lir_shr: __ asrv_w(res, x, y); break; 2591 case lir_ushr: __ lsrv_w(res, x, y); break; 2592 default: ShouldNotReachHere(); 2593 } 2594 } else if (dest->is_double_cpu()) { 2595 Register res = dest->as_register_lo(); 2596 Register x = left->as_register_lo(); 2597 Register y = count->as_register(); 2598 2599 switch (code) { 2600 case lir_shl: __ lslv(res, x, y); break; 2601 case lir_shr: __ asrv(res, x, y); break; 2602 case lir_ushr: __ lsrv(res, x, y); break; 2603 default: ShouldNotReachHere(); 2604 } 2605 } else { 2606 ShouldNotReachHere(); 2607 } 2608 #else 2609 AsmShift shift = lsl; 2610 switch (code) { 2611 case lir_shl: shift = lsl; break; 2612 case lir_shr: shift = asr; break; 2613 case lir_ushr: shift = lsr; break; 2614 default: ShouldNotReachHere(); 2615 } 2616 2617 if (dest->is_single_cpu()) { 2618 __ andr(Rtemp, count->as_register(), 31); 2619 __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, Rtemp)); 2620 } else if (dest->is_double_cpu()) { 2621 Register dest_lo = dest->as_register_lo(); 2622 Register dest_hi = dest->as_register_hi(); 2623 Register src_lo = left->as_register_lo(); 2624 Register src_hi = left->as_register_hi(); 2625 Register Rcount = count->as_register(); 2626 // Resolve possible register conflicts 2627 if (shift == lsl && dest_hi == src_lo) { 2628 dest_hi = Rtemp; 2629 } else if (shift != lsl && dest_lo == src_hi) { 2630 dest_lo = Rtemp; 2631 } else if (dest_lo == src_lo && dest_hi == src_hi) { 2632 dest_lo = Rtemp; 2633 } else if (dest_lo == Rcount || dest_hi == Rcount) { 2634 Rcount = Rtemp; 2635 } 2636 __ andr(Rcount, count->as_register(), 63); 2637 __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, Rcount); 2638 move_regs(dest_lo, dest->as_register_lo()); 2639 move_regs(dest_hi, dest->as_register_hi()); 2640 } else { 2641 ShouldNotReachHere(); 2642 } 2643 #endif // AARCH64 2644 } 2645 2646 2647 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 2648 #ifdef AARCH64 2649 if (dest->is_single_cpu()) { 2650 assert (dest->type() == T_INT, "unexpected result type"); 2651 assert (left->type() == T_INT, "unexpected left type"); 2652 count &= 31; 2653 if (count != 0) { 2654 switch (code) { 2655 case lir_shl: __ _lsl_w(dest->as_register(), left->as_register(), count); break; 2656 case lir_shr: __ _asr_w(dest->as_register(), left->as_register(), count); break; 2657 case lir_ushr: __ _lsr_w(dest->as_register(), left->as_register(), count); break; 2658 default: ShouldNotReachHere(); 2659 } 2660 } else { 2661 move_regs(left->as_register(), dest->as_register()); 2662 } 2663 } else if (dest->is_double_cpu()) { 2664 count &= 63; 2665 if (count != 0) { 2666 switch (code) { 2667 case lir_shl: __ _lsl(dest->as_register_lo(), left->as_register_lo(), count); break; 2668 case lir_shr: __ _asr(dest->as_register_lo(), left->as_register_lo(), count); break; 2669 case lir_ushr: __ _lsr(dest->as_register_lo(), left->as_register_lo(), count); break; 2670 default: ShouldNotReachHere(); 2671 } 2672 } else { 2673 move_regs(left->as_register_lo(), dest->as_register_lo()); 2674 } 2675 } else { 2676 ShouldNotReachHere(); 2677 } 2678 2679 #else 2680 AsmShift shift = lsl; 2681 switch (code) { 2682 case lir_shl: shift = lsl; break; 2683 case lir_shr: shift = asr; break; 2684 case lir_ushr: shift = lsr; break; 2685 default: ShouldNotReachHere(); 2686 } 2687 2688 if (dest->is_single_cpu()) { 2689 count &= 31; 2690 if (count != 0) { 2691 __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, count)); 2692 } else { 2693 move_regs(left->as_register(), dest->as_register()); 2694 } 2695 } else if (dest->is_double_cpu()) { 2696 count &= 63; 2697 if (count != 0) { 2698 Register dest_lo = dest->as_register_lo(); 2699 Register dest_hi = dest->as_register_hi(); 2700 Register src_lo = left->as_register_lo(); 2701 Register src_hi = left->as_register_hi(); 2702 // Resolve possible register conflicts 2703 if (shift == lsl && dest_hi == src_lo) { 2704 dest_hi = Rtemp; 2705 } else if (shift != lsl && dest_lo == src_hi) { 2706 dest_lo = Rtemp; 2707 } 2708 __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, count); 2709 move_regs(dest_lo, dest->as_register_lo()); 2710 move_regs(dest_hi, dest->as_register_hi()); 2711 } else { 2712 __ long_move(dest->as_register_lo(), dest->as_register_hi(), 2713 left->as_register_lo(), left->as_register_hi()); 2714 } 2715 } else { 2716 ShouldNotReachHere(); 2717 } 2718 #endif // AARCH64 2719 } 2720 2721 2722 // Saves 4 given registers in reserved argument area. 2723 void LIR_Assembler::save_in_reserved_area(Register r1, Register r2, Register r3, Register r4) { 2724 verify_reserved_argument_area_size(4); 2725 #ifdef AARCH64 2726 __ stp(r1, r2, Address(SP, 0)); 2727 __ stp(r3, r4, Address(SP, 2*wordSize)); 2728 #else 2729 __ stmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4)); 2730 #endif // AARCH64 2731 } 2732 2733 // Restores 4 given registers from reserved argument area. 2734 void LIR_Assembler::restore_from_reserved_area(Register r1, Register r2, Register r3, Register r4) { 2735 #ifdef AARCH64 2736 __ ldp(r1, r2, Address(SP, 0)); 2737 __ ldp(r3, r4, Address(SP, 2*wordSize)); 2738 #else 2739 __ ldmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4), no_writeback); 2740 #endif // AARCH64 2741 } 2742 2743 2744 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 2745 ciArrayKlass* default_type = op->expected_type(); 2746 Register src = op->src()->as_register(); 2747 Register src_pos = op->src_pos()->as_register(); 2748 Register dst = op->dst()->as_register(); 2749 Register dst_pos = op->dst_pos()->as_register(); 2750 Register length = op->length()->as_register(); 2751 Register tmp = op->tmp()->as_register(); 2752 Register tmp2 = Rtemp; 2753 2754 assert(src == R0 && src_pos == R1 && dst == R2 && dst_pos == R3, "code assumption"); 2755 #ifdef AARCH64 2756 assert(length == R4, "code assumption"); 2757 #endif // AARCH64 2758 2759 CodeStub* stub = op->stub(); 2760 2761 int flags = op->flags(); 2762 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL; 2763 if (basic_type == T_ARRAY) basic_type = T_OBJECT; 2764 2765 // If we don't know anything or it's an object array, just go through the generic arraycopy 2766 if (default_type == NULL) { 2767 2768 // save arguments, because they will be killed by a runtime call 2769 save_in_reserved_area(R0, R1, R2, R3); 2770 2771 #ifdef AARCH64 2772 // save length argument, will be killed by a runtime call 2773 __ raw_push(length, ZR); 2774 #else 2775 // pass length argument on SP[0] 2776 __ str(length, Address(SP, -2*wordSize, pre_indexed)); // 2 words for a proper stack alignment 2777 #endif // AARCH64 2778 2779 address copyfunc_addr = StubRoutines::generic_arraycopy(); 2780 if (copyfunc_addr == NULL) { // Use C version if stub was not generated 2781 __ call(CAST_FROM_FN_PTR(address, Runtime1::arraycopy)); 2782 } else { 2783 #ifndef PRODUCT 2784 if (PrintC1Statistics) { 2785 __ inc_counter((address)&Runtime1::_generic_arraycopystub_cnt, tmp, tmp2); 2786 } 2787 #endif // !PRODUCT 2788 // the stub is in the code cache so close enough 2789 __ call(copyfunc_addr, relocInfo::runtime_call_type); 2790 } 2791 2792 #ifdef AARCH64 2793 __ raw_pop(length, ZR); 2794 #else 2795 __ add(SP, SP, 2*wordSize); 2796 #endif // AARCH64 2797 2798 __ cbz_32(R0, *stub->continuation()); 2799 2800 if (copyfunc_addr != NULL) { 2801 __ mvn_32(tmp, R0); 2802 restore_from_reserved_area(R0, R1, R2, R3); // load saved arguments in slow case only 2803 __ sub_32(length, length, tmp); 2804 __ add_32(src_pos, src_pos, tmp); 2805 __ add_32(dst_pos, dst_pos, tmp); 2806 } else { 2807 restore_from_reserved_area(R0, R1, R2, R3); // load saved arguments in slow case only 2808 } 2809 2810 __ b(*stub->entry()); 2811 2812 __ bind(*stub->continuation()); 2813 return; 2814 } 2815 2816 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), 2817 "must be true at this point"); 2818 int elem_size = type2aelembytes(basic_type); 2819 int shift = exact_log2(elem_size); 2820 2821 // Check for NULL 2822 if (flags & LIR_OpArrayCopy::src_null_check) { 2823 if (flags & LIR_OpArrayCopy::dst_null_check) { 2824 __ cmp(src, 0); 2825 __ cond_cmp(dst, 0, ne); // make one instruction shorter if both checks are needed 2826 __ b(*stub->entry(), eq); 2827 } else { 2828 __ cbz(src, *stub->entry()); 2829 } 2830 } else if (flags & LIR_OpArrayCopy::dst_null_check) { 2831 __ cbz(dst, *stub->entry()); 2832 } 2833 2834 // If the compiler was not able to prove that exact type of the source or the destination 2835 // of the arraycopy is an array type, check at runtime if the source or the destination is 2836 // an instance type. 2837 if (flags & LIR_OpArrayCopy::type_check) { 2838 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) { 2839 __ load_klass(tmp, dst); 2840 __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2841 __ mov_slow(tmp, Klass::_lh_neutral_value); 2842 __ cmp_32(tmp2, tmp); 2843 __ b(*stub->entry(), ge); 2844 } 2845 2846 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) { 2847 __ load_klass(tmp, src); 2848 __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2849 __ mov_slow(tmp, Klass::_lh_neutral_value); 2850 __ cmp_32(tmp2, tmp); 2851 __ b(*stub->entry(), ge); 2852 } 2853 } 2854 2855 // Check if negative 2856 const int all_positive_checks = LIR_OpArrayCopy::src_pos_positive_check | 2857 LIR_OpArrayCopy::dst_pos_positive_check | 2858 LIR_OpArrayCopy::length_positive_check; 2859 switch (flags & all_positive_checks) { 2860 case LIR_OpArrayCopy::src_pos_positive_check: 2861 __ branch_if_negative_32(src_pos, *stub->entry()); 2862 break; 2863 case LIR_OpArrayCopy::dst_pos_positive_check: 2864 __ branch_if_negative_32(dst_pos, *stub->entry()); 2865 break; 2866 case LIR_OpArrayCopy::length_positive_check: 2867 __ branch_if_negative_32(length, *stub->entry()); 2868 break; 2869 case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::dst_pos_positive_check: 2870 __ branch_if_any_negative_32(src_pos, dst_pos, tmp, *stub->entry()); 2871 break; 2872 case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::length_positive_check: 2873 __ branch_if_any_negative_32(src_pos, length, tmp, *stub->entry()); 2874 break; 2875 case LIR_OpArrayCopy::dst_pos_positive_check | LIR_OpArrayCopy::length_positive_check: 2876 __ branch_if_any_negative_32(dst_pos, length, tmp, *stub->entry()); 2877 break; 2878 case all_positive_checks: 2879 __ branch_if_any_negative_32(src_pos, dst_pos, length, tmp, *stub->entry()); 2880 break; 2881 default: 2882 assert((flags & all_positive_checks) == 0, "the last option"); 2883 } 2884 2885 // Range checks 2886 if (flags & LIR_OpArrayCopy::src_range_check) { 2887 __ ldr_s32(tmp2, Address(src, arrayOopDesc::length_offset_in_bytes())); 2888 __ add_32(tmp, src_pos, length); 2889 __ cmp_32(tmp, tmp2); 2890 __ b(*stub->entry(), hi); 2891 } 2892 if (flags & LIR_OpArrayCopy::dst_range_check) { 2893 __ ldr_s32(tmp2, Address(dst, arrayOopDesc::length_offset_in_bytes())); 2894 __ add_32(tmp, dst_pos, length); 2895 __ cmp_32(tmp, tmp2); 2896 __ b(*stub->entry(), hi); 2897 } 2898 2899 // Check if src and dst are of the same type 2900 if (flags & LIR_OpArrayCopy::type_check) { 2901 // We don't know the array types are compatible 2902 if (basic_type != T_OBJECT) { 2903 // Simple test for basic type arrays 2904 if (UseCompressedClassPointers) { 2905 // We don't need decode because we just need to compare 2906 __ ldr_u32(tmp, Address(src, oopDesc::klass_offset_in_bytes())); 2907 __ ldr_u32(tmp2, Address(dst, oopDesc::klass_offset_in_bytes())); 2908 __ cmp_32(tmp, tmp2); 2909 } else { 2910 __ load_klass(tmp, src); 2911 __ load_klass(tmp2, dst); 2912 __ cmp(tmp, tmp2); 2913 } 2914 __ b(*stub->entry(), ne); 2915 } else { 2916 // For object arrays, if src is a sub class of dst then we can 2917 // safely do the copy. 2918 Label cont, slow; 2919 2920 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 2921 2922 __ load_klass(tmp, src); 2923 __ load_klass(tmp2, dst); 2924 2925 // We are at a call so all live registers are saved before we 2926 // get here 2927 assert_different_registers(tmp, tmp2, R6, altFP_7_11); 2928 2929 __ check_klass_subtype_fast_path(tmp, tmp2, R6, altFP_7_11, &cont, copyfunc_addr == NULL ? stub->entry() : &slow, NULL); 2930 2931 __ mov(R6, R0); 2932 __ mov(altFP_7_11, R1); 2933 __ mov(R0, tmp); 2934 __ mov(R1, tmp2); 2935 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); // does not blow any registers except R0, LR and Rtemp 2936 __ cmp_32(R0, 0); 2937 __ mov(R0, R6); 2938 __ mov(R1, altFP_7_11); 2939 2940 if (copyfunc_addr != NULL) { // use stub if available 2941 // src is not a sub class of dst so we have to do a 2942 // per-element check. 2943 2944 __ b(cont, ne); 2945 2946 __ bind(slow); 2947 2948 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 2949 if ((flags & mask) != mask) { 2950 // Check that at least both of them object arrays. 2951 assert(flags & mask, "one of the two should be known to be an object array"); 2952 2953 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 2954 __ load_klass(tmp, src); 2955 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 2956 __ load_klass(tmp, dst); 2957 } 2958 int lh_offset = in_bytes(Klass::layout_helper_offset()); 2959 2960 __ ldr_u32(tmp2, Address(tmp, lh_offset)); 2961 2962 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 2963 __ mov_slow(tmp, objArray_lh); 2964 __ cmp_32(tmp, tmp2); 2965 __ b(*stub->entry(), ne); 2966 } 2967 2968 save_in_reserved_area(R0, R1, R2, R3); 2969 2970 Register src_ptr = R0; 2971 Register dst_ptr = R1; 2972 Register len = R2; 2973 Register chk_off = R3; 2974 Register super_k = AARCH64_ONLY(R4) NOT_AARCH64(tmp); 2975 2976 __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type)); 2977 __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift); 2978 2979 __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type)); 2980 __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift); 2981 __ load_klass(tmp, dst); 2982 2983 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); 2984 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 2985 2986 #ifdef AARCH64 2987 __ raw_push(length, ZR); // Preserve length around *copyfunc_addr call 2988 2989 __ mov(len, length); 2990 __ ldr(super_k, Address(tmp, ek_offset)); // super_k == R4 == length, so this load cannot be performed earlier 2991 // TODO-AARCH64: check whether it is faster to load super klass early by using tmp and additional mov. 2992 __ ldr_u32(chk_off, Address(super_k, sco_offset)); 2993 #else // AARCH64 2994 __ ldr(super_k, Address(tmp, ek_offset)); 2995 2996 __ mov(len, length); 2997 __ ldr_u32(chk_off, Address(super_k, sco_offset)); 2998 __ push(super_k); 2999 #endif // AARCH64 3000 3001 __ call(copyfunc_addr, relocInfo::runtime_call_type); 3002 3003 #ifndef PRODUCT 3004 if (PrintC1Statistics) { 3005 Label failed; 3006 __ cbnz_32(R0, failed); 3007 __ inc_counter((address)&Runtime1::_arraycopy_checkcast_cnt, tmp, tmp2); 3008 __ bind(failed); 3009 } 3010 #endif // PRODUCT 3011 3012 #ifdef AARCH64 3013 __ raw_pop(length, ZR); 3014 #else 3015 __ add(SP, SP, wordSize); // Drop super_k argument 3016 #endif // AARCH64 3017 3018 __ cbz_32(R0, *stub->continuation()); 3019 __ mvn_32(tmp, R0); 3020 3021 // load saved arguments in slow case only 3022 restore_from_reserved_area(R0, R1, R2, R3); 3023 3024 __ sub_32(length, length, tmp); 3025 __ add_32(src_pos, src_pos, tmp); 3026 __ add_32(dst_pos, dst_pos, tmp); 3027 3028 #ifndef PRODUCT 3029 if (PrintC1Statistics) { 3030 __ inc_counter((address)&Runtime1::_arraycopy_checkcast_attempt_cnt, tmp, tmp2); 3031 } 3032 #endif 3033 3034 __ b(*stub->entry()); 3035 3036 __ bind(cont); 3037 } else { 3038 __ b(*stub->entry(), eq); 3039 __ bind(cont); 3040 } 3041 } 3042 } 3043 3044 #ifndef PRODUCT 3045 if (PrintC1Statistics) { 3046 address counter = Runtime1::arraycopy_count_address(basic_type); 3047 __ inc_counter(counter, tmp, tmp2); 3048 } 3049 #endif // !PRODUCT 3050 3051 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 3052 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 3053 const char *name; 3054 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 3055 3056 Register src_ptr = R0; 3057 Register dst_ptr = R1; 3058 Register len = R2; 3059 3060 __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type)); 3061 __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift); 3062 3063 __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type)); 3064 __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift); 3065 3066 __ mov(len, length); 3067 3068 __ call(entry, relocInfo::runtime_call_type); 3069 3070 __ bind(*stub->continuation()); 3071 } 3072 3073 #ifdef ASSERT 3074 // emit run-time assertion 3075 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 3076 assert(op->code() == lir_assert, "must be"); 3077 3078 #ifdef AARCH64 3079 __ NOT_IMPLEMENTED(); 3080 #else 3081 if (op->in_opr1()->is_valid()) { 3082 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 3083 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 3084 } else { 3085 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 3086 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 3087 } 3088 3089 Label ok; 3090 if (op->condition() != lir_cond_always) { 3091 AsmCondition acond; 3092 switch (op->condition()) { 3093 case lir_cond_equal: acond = eq; break; 3094 case lir_cond_notEqual: acond = ne; break; 3095 case lir_cond_less: acond = lt; break; 3096 case lir_cond_lessEqual: acond = le; break; 3097 case lir_cond_greaterEqual: acond = ge; break; 3098 case lir_cond_greater: acond = gt; break; 3099 case lir_cond_aboveEqual: acond = hs; break; 3100 case lir_cond_belowEqual: acond = ls; break; 3101 default: ShouldNotReachHere(); 3102 } 3103 __ b(ok, acond); 3104 } 3105 if (op->halt()) { 3106 const char* str = __ code_string(op->msg()); 3107 __ stop(str); 3108 } else { 3109 breakpoint(); 3110 } 3111 __ bind(ok); 3112 #endif // AARCH64 3113 } 3114 #endif // ASSERT 3115 3116 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 3117 fatal("CRC32 intrinsic is not implemented on this platform"); 3118 } 3119 3120 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 3121 Register obj = op->obj_opr()->as_pointer_register(); 3122 Register hdr = op->hdr_opr()->as_pointer_register(); 3123 Register lock = op->lock_opr()->as_pointer_register(); 3124 Register tmp = op->scratch_opr()->is_illegal() ? noreg : 3125 op->scratch_opr()->as_pointer_register(); 3126 3127 if (!UseFastLocking) { 3128 __ b(*op->stub()->entry()); 3129 } else if (op->code() == lir_lock) { 3130 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 3131 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry()); 3132 if (op->info() != NULL) { 3133 add_debug_info_for_null_check(null_check_offset, op->info()); 3134 } 3135 } else if (op->code() == lir_unlock) { 3136 __ unlock_object(hdr, obj, lock, tmp, *op->stub()->entry()); 3137 } else { 3138 ShouldNotReachHere(); 3139 } 3140 __ bind(*op->stub()->continuation()); 3141 } 3142 3143 3144 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 3145 ciMethod* method = op->profiled_method(); 3146 int bci = op->profiled_bci(); 3147 ciMethod* callee = op->profiled_callee(); 3148 3149 // Update counter for all call types 3150 ciMethodData* md = method->method_data_or_null(); 3151 assert(md != NULL, "Sanity"); 3152 ciProfileData* data = md->bci_to_data(bci); 3153 assert(data->is_CounterData(), "need CounterData for calls"); 3154 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 3155 Register mdo = op->mdo()->as_register(); 3156 assert(op->tmp1()->is_register(), "tmp1 must be allocated"); 3157 Register tmp1 = op->tmp1()->as_pointer_register(); 3158 assert_different_registers(mdo, tmp1); 3159 __ mov_metadata(mdo, md->constant_encoding()); 3160 int mdo_offset_bias = 0; 3161 int max_offset = AARCH64_ONLY(4096 << LogBytesPerWord) NOT_AARCH64(4096); 3162 if (md->byte_offset_of_slot(data, CounterData::count_offset()) + data->size_in_bytes() >= max_offset) { 3163 // The offset is large so bias the mdo by the base of the slot so 3164 // that the ldr can use an immediate offset to reference the slots of the data 3165 mdo_offset_bias = md->byte_offset_of_slot(data, CounterData::count_offset()); 3166 __ mov_slow(tmp1, mdo_offset_bias); 3167 __ add(mdo, mdo, tmp1); 3168 } 3169 3170 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias); 3171 // Perform additional virtual call profiling for invokevirtual and 3172 // invokeinterface bytecodes 3173 if (op->should_profile_receiver_type()) { 3174 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 3175 Register recv = op->recv()->as_register(); 3176 assert_different_registers(mdo, tmp1, recv); 3177 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 3178 ciKlass* known_klass = op->known_holder(); 3179 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) { 3180 // We know the type that will be seen at this call site; we can 3181 // statically update the MethodData* rather than needing to do 3182 // dynamic tests on the receiver type 3183 3184 // NOTE: we should probably put a lock around this search to 3185 // avoid collisions by concurrent compilations 3186 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 3187 uint i; 3188 for (i = 0; i < VirtualCallData::row_limit(); i++) { 3189 ciKlass* receiver = vc_data->receiver(i); 3190 if (known_klass->equals(receiver)) { 3191 Address data_addr(mdo, md->byte_offset_of_slot(data, 3192 VirtualCallData::receiver_count_offset(i)) - 3193 mdo_offset_bias); 3194 __ ldr(tmp1, data_addr); 3195 __ add(tmp1, tmp1, DataLayout::counter_increment); 3196 __ str(tmp1, data_addr); 3197 return; 3198 } 3199 } 3200 3201 // Receiver type not found in profile data; select an empty slot 3202 3203 // Note that this is less efficient than it should be because it 3204 // always does a write to the receiver part of the 3205 // VirtualCallData rather than just the first time 3206 for (i = 0; i < VirtualCallData::row_limit(); i++) { 3207 ciKlass* receiver = vc_data->receiver(i); 3208 if (receiver == NULL) { 3209 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) - 3210 mdo_offset_bias); 3211 __ mov_metadata(tmp1, known_klass->constant_encoding()); 3212 __ str(tmp1, recv_addr); 3213 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) - 3214 mdo_offset_bias); 3215 __ ldr(tmp1, data_addr); 3216 __ add(tmp1, tmp1, DataLayout::counter_increment); 3217 __ str(tmp1, data_addr); 3218 return; 3219 } 3220 } 3221 } else { 3222 __ load_klass(recv, recv); 3223 Label update_done; 3224 type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &update_done); 3225 // Receiver did not match any saved receiver and there is no empty row for it. 3226 // Increment total counter to indicate polymorphic case. 3227 __ ldr(tmp1, counter_addr); 3228 __ add(tmp1, tmp1, DataLayout::counter_increment); 3229 __ str(tmp1, counter_addr); 3230 3231 __ bind(update_done); 3232 } 3233 } else { 3234 // Static call 3235 __ ldr(tmp1, counter_addr); 3236 __ add(tmp1, tmp1, DataLayout::counter_increment); 3237 __ str(tmp1, counter_addr); 3238 } 3239 } 3240 3241 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 3242 fatal("Type profiling not implemented on this platform"); 3243 } 3244 3245 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 3246 Unimplemented(); 3247 } 3248 3249 3250 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 3251 Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no); 3252 __ add_slow(dst->as_pointer_register(), mon_addr.base(), mon_addr.disp()); 3253 } 3254 3255 3256 void LIR_Assembler::align_backward_branch_target() { 3257 // TODO-AARCH64 review it 3258 // Some ARM processors do better with 8-byte branch target alignment 3259 __ align(8); 3260 } 3261 3262 3263 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) { 3264 3265 if (left->is_single_cpu()) { 3266 assert (dest->type() == T_INT, "unexpected result type"); 3267 assert (left->type() == T_INT, "unexpected left type"); 3268 __ neg_32(dest->as_register(), left->as_register()); 3269 } else if (left->is_double_cpu()) { 3270 #ifdef AARCH64 3271 __ neg(dest->as_register_lo(), left->as_register_lo()); 3272 #else 3273 Register dest_lo = dest->as_register_lo(); 3274 Register dest_hi = dest->as_register_hi(); 3275 Register src_lo = left->as_register_lo(); 3276 Register src_hi = left->as_register_hi(); 3277 if (dest_lo == src_hi) { 3278 dest_lo = Rtemp; 3279 } 3280 __ rsbs(dest_lo, src_lo, 0); 3281 __ rsc(dest_hi, src_hi, 0); 3282 move_regs(dest_lo, dest->as_register_lo()); 3283 #endif // AARCH64 3284 } else if (left->is_single_fpu()) { 3285 __ neg_float(dest->as_float_reg(), left->as_float_reg()); 3286 } else if (left->is_double_fpu()) { 3287 __ neg_double(dest->as_double_reg(), left->as_double_reg()); 3288 } else { 3289 ShouldNotReachHere(); 3290 } 3291 } 3292 3293 3294 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest) { 3295 LIR_Address* addr = addr_opr->as_address_ptr(); 3296 if (addr->index()->is_illegal()) { 3297 jint c = addr->disp(); 3298 if (!Assembler::is_arith_imm_in_range(c)) { 3299 BAILOUT("illegal arithmetic operand"); 3300 } 3301 __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(), c); 3302 } else { 3303 assert(addr->disp() == 0, "cannot handle otherwise"); 3304 #ifdef AARCH64 3305 assert(addr->index()->is_double_cpu(), "should be"); 3306 #endif // AARCH64 3307 __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(), 3308 AsmOperand(addr->index()->as_pointer_register(), lsl, addr->scale())); 3309 } 3310 } 3311 3312 3313 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 3314 assert(!tmp->is_valid(), "don't need temporary"); 3315 __ call(dest); 3316 if (info != NULL) { 3317 add_call_info_here(info); 3318 } 3319 } 3320 3321 3322 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 3323 #ifdef AARCH64 3324 Unimplemented(); // TODO-AARCH64: Use stlr/ldar instructions for volatile load/store 3325 #else 3326 assert(src->is_double_cpu() && dest->is_address() || 3327 src->is_address() && dest->is_double_cpu(), 3328 "Simple move_op is called for all other cases"); 3329 3330 int null_check_offset; 3331 if (dest->is_address()) { 3332 // Store 3333 const LIR_Address* addr = dest->as_address_ptr(); 3334 const Register src_lo = src->as_register_lo(); 3335 const Register src_hi = src->as_register_hi(); 3336 assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already"); 3337 3338 if (src_lo < src_hi) { 3339 null_check_offset = __ offset(); 3340 __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(src_hi)); 3341 } else { 3342 assert(src_lo < Rtemp, "Rtemp is higher than any allocatable register"); 3343 __ mov(Rtemp, src_hi); 3344 null_check_offset = __ offset(); 3345 __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(Rtemp)); 3346 } 3347 } else { 3348 // Load 3349 const LIR_Address* addr = src->as_address_ptr(); 3350 const Register dest_lo = dest->as_register_lo(); 3351 const Register dest_hi = dest->as_register_hi(); 3352 assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already"); 3353 3354 null_check_offset = __ offset(); 3355 if (dest_lo < dest_hi) { 3356 __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(dest_hi)); 3357 } else { 3358 assert(dest_lo < Rtemp, "Rtemp is higher than any allocatable register"); 3359 __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(Rtemp)); 3360 __ mov(dest_hi, Rtemp); 3361 } 3362 } 3363 3364 if (info != NULL) { 3365 add_debug_info_for_null_check(null_check_offset, info); 3366 } 3367 #endif // AARCH64 3368 } 3369 3370 3371 void LIR_Assembler::membar() { 3372 __ membar(MacroAssembler::StoreLoad, Rtemp); 3373 } 3374 3375 void LIR_Assembler::membar_acquire() { 3376 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp); 3377 } 3378 3379 void LIR_Assembler::membar_release() { 3380 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp); 3381 } 3382 3383 void LIR_Assembler::membar_loadload() { 3384 __ membar(MacroAssembler::LoadLoad, Rtemp); 3385 } 3386 3387 void LIR_Assembler::membar_storestore() { 3388 __ membar(MacroAssembler::StoreStore, Rtemp); 3389 } 3390 3391 void LIR_Assembler::membar_loadstore() { 3392 __ membar(MacroAssembler::LoadStore, Rtemp); 3393 } 3394 3395 void LIR_Assembler::membar_storeload() { 3396 __ membar(MacroAssembler::StoreLoad, Rtemp); 3397 } 3398 3399 void LIR_Assembler::on_spin_wait() { 3400 Unimplemented(); 3401 } 3402 3403 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 3404 // Not used on ARM 3405 Unimplemented(); 3406 } 3407 3408 void LIR_Assembler::peephole(LIR_List* lir) { 3409 #ifdef AARCH64 3410 return; // TODO-AARCH64 implement peephole optimizations 3411 #endif 3412 LIR_OpList* inst = lir->instructions_list(); 3413 const int inst_length = inst->length(); 3414 for (int i = 0; i < inst_length; i++) { 3415 LIR_Op* op = inst->at(i); 3416 switch (op->code()) { 3417 case lir_cmp: { 3418 // Replace: 3419 // cmp rX, y 3420 // cmove [EQ] y, z, rX 3421 // with 3422 // cmp rX, y 3423 // cmove [EQ] illegalOpr, z, rX 3424 // 3425 // or 3426 // cmp rX, y 3427 // cmove [NE] z, y, rX 3428 // with 3429 // cmp rX, y 3430 // cmove [NE] z, illegalOpr, rX 3431 // 3432 // moves from illegalOpr should be removed when converting LIR to native assembly 3433 3434 LIR_Op2* cmp = op->as_Op2(); 3435 assert(cmp != NULL, "cmp LIR instruction is not an op2"); 3436 3437 if (i + 1 < inst_length) { 3438 LIR_Op2* cmove = inst->at(i + 1)->as_Op2(); 3439 if (cmove != NULL && cmove->code() == lir_cmove) { 3440 LIR_Opr cmove_res = cmove->result_opr(); 3441 bool res_is_op1 = cmove_res == cmp->in_opr1(); 3442 bool res_is_op2 = cmove_res == cmp->in_opr2(); 3443 LIR_Opr cmp_res, cmp_arg; 3444 if (res_is_op1) { 3445 cmp_res = cmp->in_opr1(); 3446 cmp_arg = cmp->in_opr2(); 3447 } else if (res_is_op2) { 3448 cmp_res = cmp->in_opr2(); 3449 cmp_arg = cmp->in_opr1(); 3450 } else { 3451 cmp_res = LIR_OprFact::illegalOpr; 3452 cmp_arg = LIR_OprFact::illegalOpr; 3453 } 3454 3455 if (cmp_res != LIR_OprFact::illegalOpr) { 3456 LIR_Condition cond = cmove->condition(); 3457 if (cond == lir_cond_equal && cmove->in_opr1() == cmp_arg) { 3458 cmove->set_in_opr1(LIR_OprFact::illegalOpr); 3459 } else if (cond == lir_cond_notEqual && cmove->in_opr2() == cmp_arg) { 3460 cmove->set_in_opr2(LIR_OprFact::illegalOpr); 3461 } 3462 } 3463 } 3464 } 3465 break; 3466 } 3467 3468 default: 3469 break; 3470 } 3471 } 3472 } 3473 3474 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) { 3475 Register ptr = src->as_pointer_register(); 3476 3477 if (code == lir_xchg) { 3478 #ifdef AARCH64 3479 if (UseCompressedOops && data->is_oop()) { 3480 __ encode_heap_oop(tmp->as_pointer_register(), data->as_register()); 3481 } 3482 #endif // AARCH64 3483 } else { 3484 assert (!data->is_oop(), "xadd for oops"); 3485 } 3486 3487 #ifndef AARCH64 3488 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp); 3489 #endif // !AARCH64 3490 3491 Label retry; 3492 __ bind(retry); 3493 3494 if ((data->type() == T_INT) || (data->is_oop() AARCH64_ONLY(&& UseCompressedOops))) { 3495 Register dst = dest->as_register(); 3496 Register new_val = noreg; 3497 #ifdef AARCH64 3498 __ ldaxr_w(dst, ptr); 3499 #else 3500 __ ldrex(dst, Address(ptr)); 3501 #endif 3502 if (code == lir_xadd) { 3503 Register tmp_reg = tmp->as_register(); 3504 if (data->is_constant()) { 3505 assert_different_registers(dst, ptr, tmp_reg); 3506 __ add_32(tmp_reg, dst, data->as_constant_ptr()->as_jint()); 3507 } else { 3508 assert_different_registers(dst, ptr, tmp_reg, data->as_register()); 3509 __ add_32(tmp_reg, dst, data->as_register()); 3510 } 3511 new_val = tmp_reg; 3512 } else { 3513 if (UseCompressedOops && data->is_oop()) { 3514 new_val = tmp->as_pointer_register(); 3515 } else { 3516 new_val = data->as_register(); 3517 } 3518 assert_different_registers(dst, ptr, new_val); 3519 } 3520 #ifdef AARCH64 3521 __ stlxr_w(Rtemp, new_val, ptr); 3522 #else 3523 __ strex(Rtemp, new_val, Address(ptr)); 3524 #endif // AARCH64 3525 3526 #ifdef AARCH64 3527 } else if ((data->type() == T_LONG) || (data->is_oop() && !UseCompressedOops)) { 3528 Register dst = dest->as_pointer_register(); 3529 Register new_val = noreg; 3530 __ ldaxr(dst, ptr); 3531 if (code == lir_xadd) { 3532 Register tmp_reg = tmp->as_pointer_register(); 3533 if (data->is_constant()) { 3534 assert_different_registers(dst, ptr, tmp_reg); 3535 jlong c = data->as_constant_ptr()->as_jlong(); 3536 assert((jlong)((jint)c) == c, "overflow"); 3537 __ add(tmp_reg, dst, (jint)c); 3538 } else { 3539 assert_different_registers(dst, ptr, tmp_reg, data->as_pointer_register()); 3540 __ add(tmp_reg, dst, data->as_pointer_register()); 3541 } 3542 new_val = tmp_reg; 3543 } else { 3544 new_val = data->as_pointer_register(); 3545 assert_different_registers(dst, ptr, new_val); 3546 } 3547 __ stlxr(Rtemp, new_val, ptr); 3548 #else 3549 } else if (data->type() == T_LONG) { 3550 Register dst_lo = dest->as_register_lo(); 3551 Register new_val_lo = noreg; 3552 Register dst_hi = dest->as_register_hi(); 3553 3554 assert(dst_hi->encoding() == dst_lo->encoding() + 1, "non aligned register pair"); 3555 assert((dst_lo->encoding() & 0x1) == 0, "misaligned register pair"); 3556 3557 __ bind(retry); 3558 __ ldrexd(dst_lo, Address(ptr)); 3559 if (code == lir_xadd) { 3560 Register tmp_lo = tmp->as_register_lo(); 3561 Register tmp_hi = tmp->as_register_hi(); 3562 3563 assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair"); 3564 assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair"); 3565 3566 if (data->is_constant()) { 3567 jlong c = data->as_constant_ptr()->as_jlong(); 3568 assert((jlong)((jint)c) == c, "overflow"); 3569 assert_different_registers(dst_lo, dst_hi, ptr, tmp_lo, tmp_hi); 3570 __ adds(tmp_lo, dst_lo, (jint)c); 3571 __ adc(tmp_hi, dst_hi, 0); 3572 } else { 3573 Register new_val_lo = data->as_register_lo(); 3574 Register new_val_hi = data->as_register_hi(); 3575 __ adds(tmp_lo, dst_lo, new_val_lo); 3576 __ adc(tmp_hi, dst_hi, new_val_hi); 3577 assert_different_registers(dst_lo, dst_hi, ptr, tmp_lo, tmp_hi, new_val_lo, new_val_hi); 3578 } 3579 new_val_lo = tmp_lo; 3580 } else { 3581 new_val_lo = data->as_register_lo(); 3582 Register new_val_hi = data->as_register_hi(); 3583 3584 assert_different_registers(dst_lo, dst_hi, ptr, new_val_lo, new_val_hi); 3585 assert(new_val_hi->encoding() == new_val_lo->encoding() + 1, "non aligned register pair"); 3586 assert((new_val_lo->encoding() & 0x1) == 0, "misaligned register pair"); 3587 } 3588 __ strexd(Rtemp, new_val_lo, Address(ptr)); 3589 #endif // AARCH64 3590 } else { 3591 ShouldNotReachHere(); 3592 } 3593 3594 __ cbnz_32(Rtemp, retry); 3595 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp); 3596 3597 #ifdef AARCH64 3598 if (UseCompressedOops && data->is_oop()) { 3599 __ decode_heap_oop(dest->as_register()); 3600 } 3601 #endif // AARCH64 3602 } 3603 3604 #undef __