1 /* 2 * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "c1/c1_Compilation.hpp" 27 #include "c1/c1_LIRAssembler.hpp" 28 #include "c1/c1_MacroAssembler.hpp" 29 #include "c1/c1_Runtime1.hpp" 30 #include "c1/c1_ValueStack.hpp" 31 #include "ci/ciArrayKlass.hpp" 32 #include "ci/ciInstance.hpp" 33 #include "gc/shared/barrierSet.hpp" 34 #include "gc/shared/cardTableModRefBS.hpp" 35 #include "gc/shared/collectedHeap.hpp" 36 #include "nativeInst_arm.hpp" 37 #include "oops/objArrayKlass.hpp" 38 #include "runtime/sharedRuntime.hpp" 39 #include "vmreg_arm.inline.hpp" 40 41 #define __ _masm-> 42 43 // Note: Rtemp usage is this file should not impact C2 and should be 44 // correct as long as it is not implicitly used in lower layers (the 45 // arm [macro]assembler) and used with care in the other C1 specific 46 // files. 47 48 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { 49 ShouldNotCallThis(); // Not used on ARM 50 return false; 51 } 52 53 54 LIR_Opr LIR_Assembler::receiverOpr() { 55 // The first register in Java calling conventions 56 return FrameMap::R0_oop_opr; 57 } 58 59 LIR_Opr LIR_Assembler::osrBufferPointer() { 60 return FrameMap::as_pointer_opr(R0); 61 } 62 63 #ifndef PRODUCT 64 void LIR_Assembler::verify_reserved_argument_area_size(int args_count) { 65 assert(args_count * wordSize <= frame_map()->reserved_argument_area_size(), "not enough space for arguments"); 66 } 67 #endif // !PRODUCT 68 69 void LIR_Assembler::store_parameter(jint c, int offset_from_sp_in_words) { 70 assert(offset_from_sp_in_words >= 0, "invalid offset from sp"); 71 int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord; 72 assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space"); 73 __ mov_slow(Rtemp, c); 74 __ str(Rtemp, Address(SP, offset_from_sp_in_bytes)); 75 } 76 77 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_sp_in_words) { 78 assert(offset_from_sp_in_words >= 0, "invalid offset from sp"); 79 int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord; 80 assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space"); 81 __ mov_metadata(Rtemp, m); 82 __ str(Rtemp, Address(SP, offset_from_sp_in_bytes)); 83 } 84 85 //--------------fpu register translations----------------------- 86 87 88 void LIR_Assembler::set_24bit_FPU() { 89 ShouldNotReachHere(); 90 } 91 92 void LIR_Assembler::reset_FPU() { 93 ShouldNotReachHere(); 94 } 95 96 void LIR_Assembler::fpop() { 97 Unimplemented(); 98 } 99 100 void LIR_Assembler::fxch(int i) { 101 Unimplemented(); 102 } 103 104 void LIR_Assembler::fld(int i) { 105 Unimplemented(); 106 } 107 108 void LIR_Assembler::ffree(int i) { 109 Unimplemented(); 110 } 111 112 void LIR_Assembler::breakpoint() { 113 __ breakpoint(); 114 } 115 116 void LIR_Assembler::push(LIR_Opr opr) { 117 Unimplemented(); 118 } 119 120 void LIR_Assembler::pop(LIR_Opr opr) { 121 Unimplemented(); 122 } 123 124 //------------------------------------------- 125 Address LIR_Assembler::as_Address(LIR_Address* addr) { 126 Register base = addr->base()->as_pointer_register(); 127 128 #ifdef AARCH64 129 int align = exact_log2(type2aelembytes(addr->type(), true)); 130 #endif 131 132 if (addr->index()->is_illegal() || addr->index()->is_constant()) { 133 int offset = addr->disp(); 134 if (addr->index()->is_constant()) { 135 offset += addr->index()->as_constant_ptr()->as_jint() << addr->scale(); 136 } 137 138 #ifdef AARCH64 139 if (!Assembler::is_unsigned_imm_in_range(offset, 12, align) && !Assembler::is_imm_in_range(offset, 9, 0)) { 140 BAILOUT_("offset not in range", Address(base)); 141 } 142 assert(UseUnalignedAccesses || (offset & right_n_bits(align)) == 0, "offset should be aligned"); 143 #else 144 if ((offset <= -4096) || (offset >= 4096)) { 145 BAILOUT_("offset not in range", Address(base)); 146 } 147 #endif // AARCH64 148 149 return Address(base, offset); 150 151 } else { 152 assert(addr->disp() == 0, "can't have both"); 153 int scale = addr->scale(); 154 155 #ifdef AARCH64 156 assert((scale == 0) || (scale == align), "scale should be zero or equal to embedded shift"); 157 158 bool is_index_extended = (addr->index()->type() == T_INT); 159 if (is_index_extended) { 160 assert(addr->index()->is_single_cpu(), "should be"); 161 return Address(base, addr->index()->as_register(), ex_sxtw, scale); 162 } else { 163 assert(addr->index()->is_double_cpu(), "should be"); 164 return Address(base, addr->index()->as_register_lo(), ex_lsl, scale); 165 } 166 #else 167 assert(addr->index()->is_single_cpu(), "should be"); 168 return scale >= 0 ? Address(base, addr->index()->as_register(), lsl, scale) : 169 Address(base, addr->index()->as_register(), lsr, -scale); 170 #endif // AARCH64 171 } 172 } 173 174 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 175 #ifdef AARCH64 176 ShouldNotCallThis(); // Not used on AArch64 177 return Address(); 178 #else 179 Address base = as_Address(addr); 180 assert(base.index() == noreg, "must be"); 181 if (base.disp() + BytesPerWord >= 4096) { BAILOUT_("offset not in range", Address(base.base(),0)); } 182 return Address(base.base(), base.disp() + BytesPerWord); 183 #endif // AARCH64 184 } 185 186 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 187 #ifdef AARCH64 188 ShouldNotCallThis(); // Not used on AArch64 189 return Address(); 190 #else 191 return as_Address(addr); 192 #endif // AARCH64 193 } 194 195 196 void LIR_Assembler::osr_entry() { 197 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 198 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 199 ValueStack* entry_state = osr_entry->end()->state(); 200 int number_of_locks = entry_state->locks_size(); 201 202 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 203 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 204 205 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 206 int monitor_offset = (method()->max_locals() + 2 * (number_of_locks - 1)) * BytesPerWord; 207 for (int i = 0; i < number_of_locks; i++) { 208 int slot_offset = monitor_offset - (i * 2 * BytesPerWord); 209 __ ldr(R1, Address(OSR_buf, slot_offset + 0*BytesPerWord)); 210 __ ldr(R2, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 211 __ str(R1, frame_map()->address_for_monitor_lock(i)); 212 __ str(R2, frame_map()->address_for_monitor_object(i)); 213 } 214 } 215 216 217 int LIR_Assembler::check_icache() { 218 Register receiver = LIR_Assembler::receiverOpr()->as_register(); 219 int offset = __ offset(); 220 __ inline_cache_check(receiver, Ricklass); 221 return offset; 222 } 223 224 225 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) { 226 jobject o = (jobject)Universe::non_oop_word(); 227 int index = __ oop_recorder()->allocate_oop_index(o); 228 229 PatchingStub* patch = new PatchingStub(_masm, patching_id(info), index); 230 231 __ patchable_mov_oop(reg, o, index); 232 patching_epilog(patch, lir_patch_normal, reg, info); 233 } 234 235 236 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 237 Metadata* o = (Metadata*)Universe::non_oop_word(); 238 int index = __ oop_recorder()->allocate_metadata_index(o); 239 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index); 240 241 __ patchable_mov_metadata(reg, o, index); 242 patching_epilog(patch, lir_patch_normal, reg, info); 243 } 244 245 246 int LIR_Assembler::initial_frame_size_in_bytes() const { 247 // Subtracts two words to account for return address and link 248 return frame_map()->framesize()*VMRegImpl::stack_slot_size - 2*wordSize; 249 } 250 251 252 int LIR_Assembler::emit_exception_handler() { 253 // TODO: ARM 254 __ nop(); // See comments in other ports 255 256 address handler_base = __ start_a_stub(exception_handler_size()); 257 if (handler_base == NULL) { 258 bailout("exception handler overflow"); 259 return -1; 260 } 261 262 int offset = code_offset(); 263 264 // check that there is really an exception 265 __ verify_not_null_oop(Rexception_obj); 266 267 __ call(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id), relocInfo::runtime_call_type); 268 __ should_not_reach_here(); 269 270 assert(code_offset() - offset <= exception_handler_size(), "overflow"); 271 __ end_a_stub(); 272 273 return offset; 274 } 275 276 // Emit the code to remove the frame from the stack in the exception 277 // unwind path. 278 int LIR_Assembler::emit_unwind_handler() { 279 #ifndef PRODUCT 280 if (CommentedAssembly) { 281 _masm->block_comment("Unwind handler"); 282 } 283 #endif 284 285 int offset = code_offset(); 286 287 // Fetch the exception from TLS and clear out exception related thread state 288 Register zero = __ zero_register(Rtemp); 289 __ ldr(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset())); 290 __ str(zero, Address(Rthread, JavaThread::exception_oop_offset())); 291 __ str(zero, Address(Rthread, JavaThread::exception_pc_offset())); 292 293 __ bind(_unwind_handler_entry); 294 __ verify_not_null_oop(Rexception_obj); 295 296 // Preform needed unlocking 297 MonitorExitStub* stub = NULL; 298 if (method()->is_synchronized()) { 299 monitor_address(0, FrameMap::R0_opr); 300 stub = new MonitorExitStub(FrameMap::R0_opr, true, 0); 301 __ unlock_object(R2, R1, R0, Rtemp, *stub->entry()); 302 __ bind(*stub->continuation()); 303 } 304 305 // remove the activation and dispatch to the unwind handler 306 __ remove_frame(initial_frame_size_in_bytes()); // restores FP and LR 307 __ jump(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type, Rtemp); 308 309 // Emit the slow path assembly 310 if (stub != NULL) { 311 stub->emit_code(this); 312 } 313 314 return offset; 315 } 316 317 318 int LIR_Assembler::emit_deopt_handler() { 319 address handler_base = __ start_a_stub(deopt_handler_size()); 320 if (handler_base == NULL) { 321 bailout("deopt handler overflow"); 322 return -1; 323 } 324 325 int offset = code_offset(); 326 327 __ mov_relative_address(LR, __ pc()); 328 #ifdef AARCH64 329 __ raw_push(LR, LR); 330 __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, Rtemp); 331 #else 332 __ push(LR); // stub expects LR to be saved 333 __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg); 334 #endif // AARCH64 335 336 assert(code_offset() - offset <= deopt_handler_size(), "overflow"); 337 __ end_a_stub(); 338 339 return offset; 340 } 341 342 343 void LIR_Assembler::return_op(LIR_Opr result) { 344 // Pop the frame before safepoint polling 345 __ remove_frame(initial_frame_size_in_bytes()); 346 347 // mov_slow here is usually one or two instruction 348 // TODO-AARCH64 3 instructions on AArch64, so try to load polling page by ldr_literal 349 __ mov_address(Rtemp, os::get_polling_page(), symbolic_Relocation::polling_page_reference); 350 __ relocate(relocInfo::poll_return_type); 351 __ ldr(Rtemp, Address(Rtemp)); 352 __ ret(); 353 } 354 355 356 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 357 __ mov_address(Rtemp, os::get_polling_page(), symbolic_Relocation::polling_page_reference); 358 if (info != NULL) { 359 add_debug_info_for_branch(info); 360 } 361 int offset = __ offset(); 362 __ relocate(relocInfo::poll_type); 363 __ ldr(Rtemp, Address(Rtemp)); 364 return offset; 365 } 366 367 368 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 369 if (from_reg != to_reg) { 370 __ mov(to_reg, from_reg); 371 } 372 } 373 374 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 375 assert(src->is_constant() && dest->is_register(), "must be"); 376 LIR_Const* c = src->as_constant_ptr(); 377 378 switch (c->type()) { 379 case T_ADDRESS: 380 case T_INT: 381 assert(patch_code == lir_patch_none, "no patching handled here"); 382 __ mov_slow(dest->as_register(), c->as_jint()); 383 break; 384 385 case T_LONG: 386 assert(patch_code == lir_patch_none, "no patching handled here"); 387 #ifdef AARCH64 388 __ mov_slow(dest->as_pointer_register(), (intptr_t)c->as_jlong()); 389 #else 390 __ mov_slow(dest->as_register_lo(), c->as_jint_lo()); 391 __ mov_slow(dest->as_register_hi(), c->as_jint_hi()); 392 #endif // AARCH64 393 break; 394 395 case T_OBJECT: 396 if (patch_code == lir_patch_none) { 397 __ mov_oop(dest->as_register(), c->as_jobject()); 398 } else { 399 jobject2reg_with_patching(dest->as_register(), info); 400 } 401 break; 402 403 case T_METADATA: 404 if (patch_code == lir_patch_none) { 405 __ mov_metadata(dest->as_register(), c->as_metadata()); 406 } else { 407 klass2reg_with_patching(dest->as_register(), info); 408 } 409 break; 410 411 case T_FLOAT: 412 if (dest->is_single_fpu()) { 413 __ mov_float(dest->as_float_reg(), c->as_jfloat()); 414 } else { 415 #ifdef AARCH64 416 ShouldNotReachHere(); 417 #else 418 // Simple getters can return float constant directly into r0 419 __ mov_slow(dest->as_register(), c->as_jint_bits()); 420 #endif // AARCH64 421 } 422 break; 423 424 case T_DOUBLE: 425 if (dest->is_double_fpu()) { 426 __ mov_double(dest->as_double_reg(), c->as_jdouble()); 427 } else { 428 #ifdef AARCH64 429 ShouldNotReachHere(); 430 #else 431 // Simple getters can return double constant directly into r1r0 432 __ mov_slow(dest->as_register_lo(), c->as_jint_lo_bits()); 433 __ mov_slow(dest->as_register_hi(), c->as_jint_hi_bits()); 434 #endif // AARCH64 435 } 436 break; 437 438 default: 439 ShouldNotReachHere(); 440 } 441 } 442 443 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 444 assert(src->is_constant(), "must be"); 445 assert(dest->is_stack(), "must be"); 446 LIR_Const* c = src->as_constant_ptr(); 447 448 switch (c->type()) { 449 case T_INT: // fall through 450 case T_FLOAT: 451 __ mov_slow(Rtemp, c->as_jint_bits()); 452 __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 453 break; 454 455 case T_ADDRESS: 456 __ mov_slow(Rtemp, c->as_jint()); 457 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 458 break; 459 460 case T_OBJECT: 461 __ mov_oop(Rtemp, c->as_jobject()); 462 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 463 break; 464 465 case T_LONG: // fall through 466 case T_DOUBLE: 467 #ifdef AARCH64 468 __ mov_slow(Rtemp, c->as_jlong_bits()); 469 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix())); 470 #else 471 __ mov_slow(Rtemp, c->as_jint_lo_bits()); 472 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes)); 473 if (c->as_jint_hi_bits() != c->as_jint_lo_bits()) { 474 __ mov_slow(Rtemp, c->as_jint_hi_bits()); 475 } 476 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes)); 477 #endif // AARCH64 478 break; 479 480 default: 481 ShouldNotReachHere(); 482 } 483 } 484 485 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, 486 CodeEmitInfo* info, bool wide) { 487 #ifdef AARCH64 488 assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL) || 489 (src->as_constant_ptr()->type() == T_INT && src->as_constant_ptr()->as_jint() == 0) || 490 (src->as_constant_ptr()->type() == T_LONG && src->as_constant_ptr()->as_jlong() == 0) || 491 (src->as_constant_ptr()->type() == T_FLOAT && src->as_constant_ptr()->as_jint_bits() == 0) || 492 (src->as_constant_ptr()->type() == T_DOUBLE && src->as_constant_ptr()->as_jlong_bits() == 0), 493 "cannot handle otherwise"); 494 assert(dest->as_address_ptr()->type() == type, "should be"); 495 496 Address addr = as_Address(dest->as_address_ptr()); 497 int null_check_offset = code_offset(); 498 switch (type) { 499 case T_OBJECT: // fall through 500 case T_ARRAY: 501 if (UseCompressedOops && !wide) { 502 __ str_w(ZR, addr); 503 } else { 504 __ str(ZR, addr); 505 } 506 break; 507 case T_ADDRESS: // fall through 508 case T_DOUBLE: // fall through 509 case T_LONG: __ str(ZR, addr); break; 510 case T_FLOAT: // fall through 511 case T_INT: __ str_w(ZR, addr); break; 512 case T_BOOLEAN: // fall through 513 case T_BYTE: __ strb(ZR, addr); break; 514 case T_CHAR: // fall through 515 case T_SHORT: __ strh(ZR, addr); break; 516 default: ShouldNotReachHere(); 517 } 518 #else 519 assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL),"cannot handle otherwise"); 520 __ mov(Rtemp, 0); 521 522 int null_check_offset = code_offset(); 523 __ str(Rtemp, as_Address(dest->as_address_ptr())); 524 #endif // AARCH64 525 526 if (info != NULL) { 527 #ifndef AARCH64 528 assert(false, "arm32 didn't support this before, investigate if bug"); 529 #endif 530 add_debug_info_for_null_check(null_check_offset, info); 531 } 532 } 533 534 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 535 assert(src->is_register() && dest->is_register(), "must be"); 536 537 if (src->is_single_cpu()) { 538 if (dest->is_single_cpu()) { 539 move_regs(src->as_register(), dest->as_register()); 540 #ifdef AARCH64 541 } else if (dest->is_double_cpu()) { 542 assert ((src->type() == T_OBJECT) || (src->type() == T_ARRAY) || (src->type() == T_ADDRESS), "invalid src type"); 543 move_regs(src->as_register(), dest->as_register_lo()); 544 #else 545 } else if (dest->is_single_fpu()) { 546 __ fmsr(dest->as_float_reg(), src->as_register()); 547 #endif // AARCH64 548 } else { 549 ShouldNotReachHere(); 550 } 551 } else if (src->is_double_cpu()) { 552 #ifdef AARCH64 553 move_regs(src->as_register_lo(), dest->as_register_lo()); 554 #else 555 if (dest->is_double_cpu()) { 556 __ long_move(dest->as_register_lo(), dest->as_register_hi(), src->as_register_lo(), src->as_register_hi()); 557 } else { 558 __ fmdrr(dest->as_double_reg(), src->as_register_lo(), src->as_register_hi()); 559 } 560 #endif // AARCH64 561 } else if (src->is_single_fpu()) { 562 if (dest->is_single_fpu()) { 563 __ mov_float(dest->as_float_reg(), src->as_float_reg()); 564 } else if (dest->is_single_cpu()) { 565 __ mov_fpr2gpr_float(dest->as_register(), src->as_float_reg()); 566 } else { 567 ShouldNotReachHere(); 568 } 569 } else if (src->is_double_fpu()) { 570 if (dest->is_double_fpu()) { 571 __ mov_double(dest->as_double_reg(), src->as_double_reg()); 572 } else if (dest->is_double_cpu()) { 573 #ifdef AARCH64 574 __ fmov_xd(dest->as_register_lo(), src->as_double_reg()); 575 #else 576 __ fmrrd(dest->as_register_lo(), dest->as_register_hi(), src->as_double_reg()); 577 #endif // AARCH64 578 } else { 579 ShouldNotReachHere(); 580 } 581 } else { 582 ShouldNotReachHere(); 583 } 584 } 585 586 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 587 assert(src->is_register(), "should not call otherwise"); 588 assert(dest->is_stack(), "should not call otherwise"); 589 590 Address addr = dest->is_single_word() ? 591 frame_map()->address_for_slot(dest->single_stack_ix()) : 592 frame_map()->address_for_slot(dest->double_stack_ix()); 593 594 #ifndef AARCH64 595 assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending"); 596 if (src->is_single_fpu() || src->is_double_fpu()) { 597 if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); } 598 } 599 #endif // !AARCH64 600 601 if (src->is_single_cpu()) { 602 switch (type) { 603 case T_OBJECT: 604 case T_ARRAY: __ verify_oop(src->as_register()); // fall through 605 case T_ADDRESS: 606 case T_METADATA: __ str(src->as_register(), addr); break; 607 case T_FLOAT: // used in intBitsToFloat intrinsic implementation, fall through 608 case T_INT: __ str_32(src->as_register(), addr); break; 609 default: 610 ShouldNotReachHere(); 611 } 612 } else if (src->is_double_cpu()) { 613 __ str(src->as_register_lo(), addr); 614 #ifndef AARCH64 615 __ str(src->as_register_hi(), frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes)); 616 #endif // !AARCH64 617 } else if (src->is_single_fpu()) { 618 __ str_float(src->as_float_reg(), addr); 619 } else if (src->is_double_fpu()) { 620 __ str_double(src->as_double_reg(), addr); 621 } else { 622 ShouldNotReachHere(); 623 } 624 } 625 626 627 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, 628 LIR_PatchCode patch_code, CodeEmitInfo* info, 629 bool pop_fpu_stack, bool wide, 630 bool unaligned) { 631 LIR_Address* to_addr = dest->as_address_ptr(); 632 Register base_reg = to_addr->base()->as_pointer_register(); 633 const bool needs_patching = (patch_code != lir_patch_none); 634 635 PatchingStub* patch = NULL; 636 if (needs_patching) { 637 #ifdef AARCH64 638 // Same alignment of reg2mem code and PatchingStub code. Required to make copied bind_literal() code properly aligned. 639 __ align(wordSize); 640 #endif 641 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 642 #ifdef AARCH64 643 // Extra nop for MT safe patching 644 __ nop(); 645 #endif // AARCH64 646 } 647 648 int null_check_offset = code_offset(); 649 650 switch (type) { 651 case T_ARRAY: 652 case T_OBJECT: 653 if (UseCompressedOops && !wide) { 654 #ifdef AARCH64 655 const Register temp_src = Rtemp; 656 assert_different_registers(temp_src, src->as_register()); 657 __ encode_heap_oop(temp_src, src->as_register()); 658 null_check_offset = code_offset(); 659 __ str_32(temp_src, as_Address(to_addr)); 660 #else 661 ShouldNotReachHere(); 662 #endif // AARCH64 663 } else { 664 __ str(src->as_register(), as_Address(to_addr)); 665 } 666 break; 667 668 case T_ADDRESS: 669 #ifdef AARCH64 670 case T_LONG: 671 #endif // AARCH64 672 __ str(src->as_pointer_register(), as_Address(to_addr)); 673 break; 674 675 case T_BYTE: 676 case T_BOOLEAN: 677 __ strb(src->as_register(), as_Address(to_addr)); 678 break; 679 680 case T_CHAR: 681 case T_SHORT: 682 __ strh(src->as_register(), as_Address(to_addr)); 683 break; 684 685 case T_INT: 686 #ifdef __SOFTFP__ 687 case T_FLOAT: 688 #endif // __SOFTFP__ 689 __ str_32(src->as_register(), as_Address(to_addr)); 690 break; 691 692 #ifdef AARCH64 693 694 case T_FLOAT: 695 __ str_s(src->as_float_reg(), as_Address(to_addr)); 696 break; 697 698 case T_DOUBLE: 699 __ str_d(src->as_double_reg(), as_Address(to_addr)); 700 break; 701 702 #else // AARCH64 703 704 #ifdef __SOFTFP__ 705 case T_DOUBLE: 706 #endif // __SOFTFP__ 707 case T_LONG: { 708 Register from_lo = src->as_register_lo(); 709 Register from_hi = src->as_register_hi(); 710 if (to_addr->index()->is_register()) { 711 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 712 assert(to_addr->disp() == 0, "Not yet supporting both"); 713 __ add(Rtemp, base_reg, to_addr->index()->as_register()); 714 base_reg = Rtemp; 715 __ str(from_lo, Address(Rtemp)); 716 if (patch != NULL) { 717 patching_epilog(patch, lir_patch_low, base_reg, info); 718 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 719 patch_code = lir_patch_high; 720 } 721 __ str(from_hi, Address(Rtemp, BytesPerWord)); 722 } else if (base_reg == from_lo) { 723 __ str(from_hi, as_Address_hi(to_addr)); 724 if (patch != NULL) { 725 patching_epilog(patch, lir_patch_high, base_reg, info); 726 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 727 patch_code = lir_patch_low; 728 } 729 __ str(from_lo, as_Address_lo(to_addr)); 730 } else { 731 __ str(from_lo, as_Address_lo(to_addr)); 732 if (patch != NULL) { 733 patching_epilog(patch, lir_patch_low, base_reg, info); 734 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 735 patch_code = lir_patch_high; 736 } 737 __ str(from_hi, as_Address_hi(to_addr)); 738 } 739 break; 740 } 741 742 #ifndef __SOFTFP__ 743 case T_FLOAT: 744 if (to_addr->index()->is_register()) { 745 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 746 __ add(Rtemp, base_reg, to_addr->index()->as_register()); 747 if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); } 748 __ fsts(src->as_float_reg(), Address(Rtemp, to_addr->disp())); 749 } else { 750 __ fsts(src->as_float_reg(), as_Address(to_addr)); 751 } 752 break; 753 754 case T_DOUBLE: 755 if (to_addr->index()->is_register()) { 756 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 757 __ add(Rtemp, base_reg, to_addr->index()->as_register()); 758 if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); } 759 __ fstd(src->as_double_reg(), Address(Rtemp, to_addr->disp())); 760 } else { 761 __ fstd(src->as_double_reg(), as_Address(to_addr)); 762 } 763 break; 764 #endif // __SOFTFP__ 765 766 #endif // AARCH64 767 768 default: 769 ShouldNotReachHere(); 770 } 771 772 if (info != NULL) { 773 add_debug_info_for_null_check(null_check_offset, info); 774 } 775 776 if (patch != NULL) { 777 // Offset embeedded into LDR/STR instruction may appear not enough 778 // to address a field. So, provide a space for one more instruction 779 // that will deal with larger offsets. 780 __ nop(); 781 patching_epilog(patch, patch_code, base_reg, info); 782 } 783 } 784 785 786 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 787 assert(src->is_stack(), "should not call otherwise"); 788 assert(dest->is_register(), "should not call otherwise"); 789 790 Address addr = src->is_single_word() ? 791 frame_map()->address_for_slot(src->single_stack_ix()) : 792 frame_map()->address_for_slot(src->double_stack_ix()); 793 794 #ifndef AARCH64 795 assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending"); 796 if (dest->is_single_fpu() || dest->is_double_fpu()) { 797 if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); } 798 } 799 #endif // !AARCH64 800 801 if (dest->is_single_cpu()) { 802 switch (type) { 803 case T_OBJECT: 804 case T_ARRAY: 805 case T_ADDRESS: 806 case T_METADATA: __ ldr(dest->as_register(), addr); break; 807 case T_FLOAT: // used in floatToRawIntBits intrinsic implemenation 808 case T_INT: __ ldr_u32(dest->as_register(), addr); break; 809 default: 810 ShouldNotReachHere(); 811 } 812 if ((type == T_OBJECT) || (type == T_ARRAY)) { 813 __ verify_oop(dest->as_register()); 814 } 815 } else if (dest->is_double_cpu()) { 816 __ ldr(dest->as_register_lo(), addr); 817 #ifndef AARCH64 818 __ ldr(dest->as_register_hi(), frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes)); 819 #endif // !AARCH64 820 } else if (dest->is_single_fpu()) { 821 __ ldr_float(dest->as_float_reg(), addr); 822 } else if (dest->is_double_fpu()) { 823 __ ldr_double(dest->as_double_reg(), addr); 824 } else { 825 ShouldNotReachHere(); 826 } 827 } 828 829 830 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 831 if (src->is_single_stack()) { 832 switch (src->type()) { 833 case T_OBJECT: 834 case T_ARRAY: 835 case T_ADDRESS: 836 case T_METADATA: 837 __ ldr(Rtemp, frame_map()->address_for_slot(src->single_stack_ix())); 838 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 839 break; 840 841 case T_INT: 842 case T_FLOAT: 843 __ ldr_u32(Rtemp, frame_map()->address_for_slot(src->single_stack_ix())); 844 __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix())); 845 break; 846 847 default: 848 ShouldNotReachHere(); 849 } 850 } else { 851 assert(src->is_double_stack(), "must be"); 852 __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes)); 853 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes)); 854 #ifdef AARCH64 855 assert(lo_word_offset_in_bytes == 0, "adjust this code"); 856 #else 857 __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes)); 858 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes)); 859 #endif // AARCH64 860 } 861 } 862 863 864 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, 865 LIR_PatchCode patch_code, CodeEmitInfo* info, 866 bool wide, bool unaligned) { 867 assert(src->is_address(), "should not call otherwise"); 868 assert(dest->is_register(), "should not call otherwise"); 869 LIR_Address* addr = src->as_address_ptr(); 870 871 Register base_reg = addr->base()->as_pointer_register(); 872 873 PatchingStub* patch = NULL; 874 if (patch_code != lir_patch_none) { 875 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 876 #ifdef AARCH64 877 // Extra nop for MT safe patching 878 __ nop(); 879 #endif // AARCH64 880 } 881 if (info != NULL) { 882 add_debug_info_for_null_check_here(info); 883 } 884 885 switch (type) { 886 case T_OBJECT: // fall through 887 case T_ARRAY: 888 if (UseCompressedOops && !wide) { 889 __ ldr_u32(dest->as_register(), as_Address(addr)); 890 } else { 891 __ ldr(dest->as_register(), as_Address(addr)); 892 } 893 break; 894 895 case T_ADDRESS: 896 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { 897 __ ldr_u32(dest->as_pointer_register(), as_Address(addr)); 898 } else { 899 __ ldr(dest->as_pointer_register(), as_Address(addr)); 900 } 901 break; 902 903 #ifdef AARCH64 904 case T_LONG: 905 #else 906 case T_INT: 907 #ifdef __SOFTFP__ 908 case T_FLOAT: 909 #endif // __SOFTFP__ 910 #endif // AARCH64 911 __ ldr(dest->as_pointer_register(), as_Address(addr)); 912 break; 913 914 case T_BOOLEAN: 915 __ ldrb(dest->as_register(), as_Address(addr)); 916 break; 917 918 case T_BYTE: 919 __ ldrsb(dest->as_register(), as_Address(addr)); 920 break; 921 922 case T_CHAR: 923 __ ldrh(dest->as_register(), as_Address(addr)); 924 break; 925 926 case T_SHORT: 927 __ ldrsh(dest->as_register(), as_Address(addr)); 928 break; 929 930 #ifdef AARCH64 931 932 case T_INT: 933 __ ldr_w(dest->as_register(), as_Address(addr)); 934 break; 935 936 case T_FLOAT: 937 __ ldr_s(dest->as_float_reg(), as_Address(addr)); 938 break; 939 940 case T_DOUBLE: 941 __ ldr_d(dest->as_double_reg(), as_Address(addr)); 942 break; 943 944 #else // AARCH64 945 946 #ifdef __SOFTFP__ 947 case T_DOUBLE: 948 #endif // __SOFTFP__ 949 case T_LONG: { 950 Register to_lo = dest->as_register_lo(); 951 Register to_hi = dest->as_register_hi(); 952 if (addr->index()->is_register()) { 953 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 954 assert(addr->disp() == 0, "Not yet supporting both"); 955 __ add(Rtemp, base_reg, addr->index()->as_register()); 956 base_reg = Rtemp; 957 __ ldr(to_lo, Address(Rtemp)); 958 if (patch != NULL) { 959 patching_epilog(patch, lir_patch_low, base_reg, info); 960 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 961 patch_code = lir_patch_high; 962 } 963 __ ldr(to_hi, Address(Rtemp, BytesPerWord)); 964 } else if (base_reg == to_lo) { 965 __ ldr(to_hi, as_Address_hi(addr)); 966 if (patch != NULL) { 967 patching_epilog(patch, lir_patch_high, base_reg, info); 968 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 969 patch_code = lir_patch_low; 970 } 971 __ ldr(to_lo, as_Address_lo(addr)); 972 } else { 973 __ ldr(to_lo, as_Address_lo(addr)); 974 if (patch != NULL) { 975 patching_epilog(patch, lir_patch_low, base_reg, info); 976 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 977 patch_code = lir_patch_high; 978 } 979 __ ldr(to_hi, as_Address_hi(addr)); 980 } 981 break; 982 } 983 984 #ifndef __SOFTFP__ 985 case T_FLOAT: 986 if (addr->index()->is_register()) { 987 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 988 __ add(Rtemp, base_reg, addr->index()->as_register()); 989 if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); } 990 __ flds(dest->as_float_reg(), Address(Rtemp, addr->disp())); 991 } else { 992 __ flds(dest->as_float_reg(), as_Address(addr)); 993 } 994 break; 995 996 case T_DOUBLE: 997 if (addr->index()->is_register()) { 998 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register"); 999 __ add(Rtemp, base_reg, addr->index()->as_register()); 1000 if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); } 1001 __ fldd(dest->as_double_reg(), Address(Rtemp, addr->disp())); 1002 } else { 1003 __ fldd(dest->as_double_reg(), as_Address(addr)); 1004 } 1005 break; 1006 #endif // __SOFTFP__ 1007 1008 #endif // AARCH64 1009 1010 default: 1011 ShouldNotReachHere(); 1012 } 1013 1014 if (patch != NULL) { 1015 // Offset embeedded into LDR/STR instruction may appear not enough 1016 // to address a field. So, provide a space for one more instruction 1017 // that will deal with larger offsets. 1018 __ nop(); 1019 patching_epilog(patch, patch_code, base_reg, info); 1020 } 1021 1022 #ifdef AARCH64 1023 switch (type) { 1024 case T_ARRAY: 1025 case T_OBJECT: 1026 if (UseCompressedOops && !wide) { 1027 __ decode_heap_oop(dest->as_register()); 1028 } 1029 __ verify_oop(dest->as_register()); 1030 break; 1031 1032 case T_ADDRESS: 1033 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1034 __ decode_klass_not_null(dest->as_register()); 1035 } 1036 break; 1037 } 1038 #endif // AARCH64 1039 } 1040 1041 1042 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1043 bool is_32 = op->result_opr()->is_single_cpu(); 1044 1045 if (op->code() == lir_idiv && op->in_opr2()->is_constant() && is_32) { 1046 int c = op->in_opr2()->as_constant_ptr()->as_jint(); 1047 assert(is_power_of_2(c), "non power-of-2 constant should be put in a register"); 1048 1049 Register left = op->in_opr1()->as_register(); 1050 Register dest = op->result_opr()->as_register(); 1051 if (c == 1) { 1052 __ mov(dest, left); 1053 } else if (c == 2) { 1054 __ add_32(dest, left, AsmOperand(left, lsr, 31)); 1055 __ asr_32(dest, dest, 1); 1056 } else if (c != (int) 0x80000000) { 1057 int power = log2_intptr(c); 1058 __ asr_32(Rtemp, left, 31); 1059 __ add_32(dest, left, AsmOperand(Rtemp, lsr, 32-power)); // dest = left + (left < 0 ? 2^power - 1 : 0); 1060 __ asr_32(dest, dest, power); // dest = dest >>> power; 1061 } else { 1062 // x/0x80000000 is a special case, since dividend is a power of two, but is negative. 1063 // The only possible result values are 0 and 1, with 1 only for dividend == divisor == 0x80000000. 1064 __ cmp_32(left, c); 1065 #ifdef AARCH64 1066 __ cset(dest, eq); 1067 #else 1068 __ mov(dest, 0, ne); 1069 __ mov(dest, 1, eq); 1070 #endif // AARCH64 1071 } 1072 } else { 1073 #ifdef AARCH64 1074 Register left = op->in_opr1()->as_pointer_register(); 1075 Register right = op->in_opr2()->as_pointer_register(); 1076 Register dest = op->result_opr()->as_pointer_register(); 1077 1078 switch (op->code()) { 1079 case lir_idiv: 1080 if (is_32) { 1081 __ sdiv_w(dest, left, right); 1082 } else { 1083 __ sdiv(dest, left, right); 1084 } 1085 break; 1086 case lir_irem: { 1087 Register tmp = op->in_opr3()->as_pointer_register(); 1088 assert_different_registers(left, tmp); 1089 assert_different_registers(right, tmp); 1090 if (is_32) { 1091 __ sdiv_w(tmp, left, right); 1092 __ msub_w(dest, right, tmp, left); 1093 } else { 1094 __ sdiv(tmp, left, right); 1095 __ msub(dest, right, tmp, left); 1096 } 1097 break; 1098 } 1099 default: 1100 ShouldNotReachHere(); 1101 } 1102 #else 1103 assert(op->code() == lir_idiv || op->code() == lir_irem, "unexpected op3"); 1104 __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type); 1105 add_debug_info_for_div0_here(op->info()); 1106 #endif // AARCH64 1107 } 1108 } 1109 1110 1111 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1112 #ifdef ASSERT 1113 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label"); 1114 if (op->block() != NULL) _branch_target_blocks.append(op->block()); 1115 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock()); 1116 assert(op->info() == NULL, "CodeEmitInfo?"); 1117 #endif // ASSERT 1118 1119 #ifdef __SOFTFP__ 1120 assert (op->code() != lir_cond_float_branch, "this should be impossible"); 1121 #else 1122 if (op->code() == lir_cond_float_branch) { 1123 #ifndef AARCH64 1124 __ fmstat(); 1125 #endif // !AARCH64 1126 __ b(*(op->ublock()->label()), vs); 1127 } 1128 #endif // __SOFTFP__ 1129 1130 AsmCondition acond = al; 1131 switch (op->cond()) { 1132 case lir_cond_equal: acond = eq; break; 1133 case lir_cond_notEqual: acond = ne; break; 1134 case lir_cond_less: acond = lt; break; 1135 case lir_cond_lessEqual: acond = le; break; 1136 case lir_cond_greaterEqual: acond = ge; break; 1137 case lir_cond_greater: acond = gt; break; 1138 case lir_cond_aboveEqual: acond = hs; break; 1139 case lir_cond_belowEqual: acond = ls; break; 1140 default: assert(op->cond() == lir_cond_always, "must be"); 1141 } 1142 __ b(*(op->label()), acond); 1143 } 1144 1145 1146 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1147 LIR_Opr src = op->in_opr(); 1148 LIR_Opr dest = op->result_opr(); 1149 1150 switch (op->bytecode()) { 1151 case Bytecodes::_i2l: 1152 #ifdef AARCH64 1153 __ sign_extend(dest->as_register_lo(), src->as_register(), 32); 1154 #else 1155 move_regs(src->as_register(), dest->as_register_lo()); 1156 __ mov(dest->as_register_hi(), AsmOperand(src->as_register(), asr, 31)); 1157 #endif // AARCH64 1158 break; 1159 case Bytecodes::_l2i: 1160 move_regs(src->as_register_lo(), dest->as_register()); 1161 break; 1162 case Bytecodes::_i2b: 1163 __ sign_extend(dest->as_register(), src->as_register(), 8); 1164 break; 1165 case Bytecodes::_i2s: 1166 __ sign_extend(dest->as_register(), src->as_register(), 16); 1167 break; 1168 case Bytecodes::_i2c: 1169 __ zero_extend(dest->as_register(), src->as_register(), 16); 1170 break; 1171 case Bytecodes::_f2d: 1172 __ convert_f2d(dest->as_double_reg(), src->as_float_reg()); 1173 break; 1174 case Bytecodes::_d2f: 1175 __ convert_d2f(dest->as_float_reg(), src->as_double_reg()); 1176 break; 1177 case Bytecodes::_i2f: 1178 #ifdef AARCH64 1179 __ scvtf_sw(dest->as_float_reg(), src->as_register()); 1180 #else 1181 __ fmsr(Stemp, src->as_register()); 1182 __ fsitos(dest->as_float_reg(), Stemp); 1183 #endif // AARCH64 1184 break; 1185 case Bytecodes::_i2d: 1186 #ifdef AARCH64 1187 __ scvtf_dw(dest->as_double_reg(), src->as_register()); 1188 #else 1189 __ fmsr(Stemp, src->as_register()); 1190 __ fsitod(dest->as_double_reg(), Stemp); 1191 #endif // AARCH64 1192 break; 1193 case Bytecodes::_f2i: 1194 #ifdef AARCH64 1195 __ fcvtzs_ws(dest->as_register(), src->as_float_reg()); 1196 #else 1197 __ ftosizs(Stemp, src->as_float_reg()); 1198 __ fmrs(dest->as_register(), Stemp); 1199 #endif // AARCH64 1200 break; 1201 case Bytecodes::_d2i: 1202 #ifdef AARCH64 1203 __ fcvtzs_wd(dest->as_register(), src->as_double_reg()); 1204 #else 1205 __ ftosizd(Stemp, src->as_double_reg()); 1206 __ fmrs(dest->as_register(), Stemp); 1207 #endif // AARCH64 1208 break; 1209 #ifdef AARCH64 1210 case Bytecodes::_l2f: 1211 __ scvtf_sx(dest->as_float_reg(), src->as_register_lo()); 1212 break; 1213 case Bytecodes::_l2d: 1214 __ scvtf_dx(dest->as_double_reg(), src->as_register_lo()); 1215 break; 1216 case Bytecodes::_f2l: 1217 __ fcvtzs_xs(dest->as_register_lo(), src->as_float_reg()); 1218 break; 1219 case Bytecodes::_d2l: 1220 __ fcvtzs_xd(dest->as_register_lo(), src->as_double_reg()); 1221 break; 1222 #endif // AARCH64 1223 default: 1224 ShouldNotReachHere(); 1225 } 1226 } 1227 1228 1229 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1230 if (op->init_check()) { 1231 Register tmp = op->tmp1()->as_register(); 1232 __ ldrb(tmp, Address(op->klass()->as_register(), InstanceKlass::init_state_offset())); 1233 add_debug_info_for_null_check_here(op->stub()->info()); 1234 __ cmp(tmp, InstanceKlass::fully_initialized); 1235 __ b(*op->stub()->entry(), ne); 1236 } 1237 __ allocate_object(op->obj()->as_register(), 1238 op->tmp1()->as_register(), 1239 op->tmp2()->as_register(), 1240 op->tmp3()->as_register(), 1241 op->header_size(), 1242 op->object_size(), 1243 op->klass()->as_register(), 1244 *op->stub()->entry()); 1245 __ bind(*op->stub()->continuation()); 1246 } 1247 1248 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1249 if (UseSlowPath || 1250 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || 1251 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { 1252 __ b(*op->stub()->entry()); 1253 } else { 1254 __ allocate_array(op->obj()->as_register(), 1255 op->len()->as_register(), 1256 op->tmp1()->as_register(), 1257 op->tmp2()->as_register(), 1258 op->tmp3()->as_register(), 1259 arrayOopDesc::header_size(op->type()), 1260 type2aelembytes(op->type()), 1261 op->klass()->as_register(), 1262 *op->stub()->entry()); 1263 } 1264 __ bind(*op->stub()->continuation()); 1265 } 1266 1267 void LIR_Assembler::type_profile_helper(Register mdo, int mdo_offset_bias, 1268 ciMethodData *md, ciProfileData *data, 1269 Register recv, Register tmp1, Label* update_done) { 1270 assert_different_registers(mdo, recv, tmp1); 1271 uint i; 1272 for (i = 0; i < VirtualCallData::row_limit(); i++) { 1273 Label next_test; 1274 // See if the receiver is receiver[n]. 1275 Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) - 1276 mdo_offset_bias); 1277 __ ldr(tmp1, receiver_addr); 1278 __ verify_klass_ptr(tmp1); 1279 __ cmp(recv, tmp1); 1280 __ b(next_test, ne); 1281 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) - 1282 mdo_offset_bias); 1283 __ ldr(tmp1, data_addr); 1284 __ add(tmp1, tmp1, DataLayout::counter_increment); 1285 __ str(tmp1, data_addr); 1286 __ b(*update_done); 1287 __ bind(next_test); 1288 } 1289 1290 // Didn't find receiver; find next empty slot and fill it in 1291 for (i = 0; i < VirtualCallData::row_limit(); i++) { 1292 Label next_test; 1293 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) - 1294 mdo_offset_bias); 1295 __ ldr(tmp1, recv_addr); 1296 __ cbnz(tmp1, next_test); 1297 __ str(recv, recv_addr); 1298 __ mov(tmp1, DataLayout::counter_increment); 1299 __ str(tmp1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) - 1300 mdo_offset_bias)); 1301 __ b(*update_done); 1302 __ bind(next_test); 1303 } 1304 } 1305 1306 void LIR_Assembler::setup_md_access(ciMethod* method, int bci, 1307 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) { 1308 md = method->method_data_or_null(); 1309 assert(md != NULL, "Sanity"); 1310 data = md->bci_to_data(bci); 1311 assert(data != NULL, "need data for checkcast"); 1312 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1313 if (md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes() >= 4096) { 1314 // The offset is large so bias the mdo by the base of the slot so 1315 // that the ldr can use an immediate offset to reference the slots of the data 1316 mdo_offset_bias = md->byte_offset_of_slot(data, DataLayout::header_offset()); 1317 } 1318 } 1319 1320 // On 32-bit ARM, code before this helper should test obj for null (ZF should be set if obj is null). 1321 void LIR_Assembler::typecheck_profile_helper1(ciMethod* method, int bci, 1322 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias, 1323 Register obj, Register mdo, Register data_val, Label* obj_is_null) { 1324 assert(method != NULL, "Should have method"); 1325 assert_different_registers(obj, mdo, data_val); 1326 setup_md_access(method, bci, md, data, mdo_offset_bias); 1327 Label not_null; 1328 #ifdef AARCH64 1329 __ cbnz(obj, not_null); 1330 #else 1331 __ b(not_null, ne); 1332 #endif // AARCH64 1333 __ mov_metadata(mdo, md->constant_encoding()); 1334 if (mdo_offset_bias > 0) { 1335 __ mov_slow(data_val, mdo_offset_bias); 1336 __ add(mdo, mdo, data_val); 1337 } 1338 Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias); 1339 __ ldrb(data_val, flags_addr); 1340 __ orr(data_val, data_val, (uint)BitData::null_seen_byte_constant()); 1341 __ strb(data_val, flags_addr); 1342 __ b(*obj_is_null); 1343 __ bind(not_null); 1344 } 1345 1346 void LIR_Assembler::typecheck_profile_helper2(ciMethodData* md, ciProfileData* data, int mdo_offset_bias, 1347 Register mdo, Register recv, Register value, Register tmp1, 1348 Label* profile_cast_success, Label* profile_cast_failure, 1349 Label* success, Label* failure) { 1350 assert_different_registers(mdo, value, tmp1); 1351 __ bind(*profile_cast_success); 1352 __ mov_metadata(mdo, md->constant_encoding()); 1353 if (mdo_offset_bias > 0) { 1354 __ mov_slow(tmp1, mdo_offset_bias); 1355 __ add(mdo, mdo, tmp1); 1356 } 1357 __ load_klass(recv, value); 1358 type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, success); 1359 __ b(*success); 1360 // Cast failure case 1361 __ bind(*profile_cast_failure); 1362 __ mov_metadata(mdo, md->constant_encoding()); 1363 if (mdo_offset_bias > 0) { 1364 __ mov_slow(tmp1, mdo_offset_bias); 1365 __ add(mdo, mdo, tmp1); 1366 } 1367 Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias); 1368 __ ldr(tmp1, data_addr); 1369 __ sub(tmp1, tmp1, DataLayout::counter_increment); 1370 __ str(tmp1, data_addr); 1371 __ b(*failure); 1372 } 1373 1374 // Sets `res` to true, if `cond` holds. On AArch64 also sets `res` to false if `cond` does not hold. 1375 static void set_instanceof_result(MacroAssembler* _masm, Register res, AsmCondition cond) { 1376 #ifdef AARCH64 1377 __ cset(res, cond); 1378 #else 1379 __ mov(res, 1, cond); 1380 #endif // AARCH64 1381 } 1382 1383 1384 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1385 // TODO: ARM - can be more effective with one more register 1386 switch (op->code()) { 1387 case lir_store_check: { 1388 CodeStub* stub = op->stub(); 1389 Register value = op->object()->as_register(); 1390 Register array = op->array()->as_register(); 1391 Register klass_RInfo = op->tmp1()->as_register(); 1392 Register k_RInfo = op->tmp2()->as_register(); 1393 assert_different_registers(klass_RInfo, k_RInfo, Rtemp); 1394 if (op->should_profile()) { 1395 assert_different_registers(value, klass_RInfo, k_RInfo, Rtemp); 1396 } 1397 1398 // check if it needs to be profiled 1399 ciMethodData* md; 1400 ciProfileData* data; 1401 int mdo_offset_bias = 0; 1402 Label profile_cast_success, profile_cast_failure, done; 1403 Label *success_target = op->should_profile() ? &profile_cast_success : &done; 1404 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry(); 1405 1406 if (op->should_profile()) { 1407 #ifndef AARCH64 1408 __ cmp(value, 0); 1409 #endif // !AARCH64 1410 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, value, k_RInfo, Rtemp, &done); 1411 } else { 1412 __ cbz(value, done); 1413 } 1414 assert_different_registers(k_RInfo, value); 1415 add_debug_info_for_null_check_here(op->info_for_exception()); 1416 __ load_klass(k_RInfo, array); 1417 __ load_klass(klass_RInfo, value); 1418 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1419 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1420 // check for immediate positive hit 1421 __ ldr(Rtemp, Address(klass_RInfo, Rtemp)); 1422 __ cmp(klass_RInfo, k_RInfo); 1423 __ cond_cmp(Rtemp, k_RInfo, ne); 1424 __ b(*success_target, eq); 1425 // check for immediate negative hit 1426 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1427 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset())); 1428 __ b(*failure_target, ne); 1429 // slow case 1430 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1431 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1432 __ cbz(R0, *failure_target); 1433 if (op->should_profile()) { 1434 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp; 1435 if (mdo == value) { 1436 mdo = k_RInfo; 1437 recv = klass_RInfo; 1438 } 1439 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, value, tmp1, 1440 &profile_cast_success, &profile_cast_failure, 1441 &done, stub->entry()); 1442 } 1443 __ bind(done); 1444 break; 1445 } 1446 1447 case lir_checkcast: { 1448 CodeStub* stub = op->stub(); 1449 Register obj = op->object()->as_register(); 1450 Register res = op->result_opr()->as_register(); 1451 Register klass_RInfo = op->tmp1()->as_register(); 1452 Register k_RInfo = op->tmp2()->as_register(); 1453 ciKlass* k = op->klass(); 1454 assert_different_registers(res, k_RInfo, klass_RInfo, Rtemp); 1455 1456 // TODO: ARM - Late binding is used to prevent confusion of register allocator 1457 assert(stub->is_exception_throw_stub(), "must be"); 1458 ((SimpleExceptionStub*)stub)->set_obj(op->result_opr()); 1459 1460 ciMethodData* md; 1461 ciProfileData* data; 1462 int mdo_offset_bias = 0; 1463 1464 Label done; 1465 1466 Label profile_cast_failure, profile_cast_success; 1467 Label *failure_target = op->should_profile() ? &profile_cast_failure : op->stub()->entry(); 1468 Label *success_target = op->should_profile() ? &profile_cast_success : &done; 1469 1470 #ifdef AARCH64 1471 move_regs(obj, res); 1472 if (op->should_profile()) { 1473 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done); 1474 } else { 1475 __ cbz(obj, done); 1476 } 1477 if (k->is_loaded()) { 1478 __ mov_metadata(k_RInfo, k->constant_encoding()); 1479 } else { 1480 if (res != obj) { 1481 op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res)); 1482 } 1483 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1484 } 1485 __ load_klass(klass_RInfo, res); 1486 1487 if (op->fast_check()) { 1488 __ cmp(klass_RInfo, k_RInfo); 1489 __ b(*failure_target, ne); 1490 } else if (k->is_loaded()) { 1491 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset())); 1492 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) { 1493 __ cmp(Rtemp, k_RInfo); 1494 __ b(*failure_target, ne); 1495 } else { 1496 __ cmp(klass_RInfo, k_RInfo); 1497 __ cond_cmp(Rtemp, k_RInfo, ne); 1498 __ b(*success_target, eq); 1499 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1500 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1501 __ cbz(R0, *failure_target); 1502 } 1503 } else { 1504 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1505 // check for immediate positive hit 1506 __ ldr(Rtemp, Address(klass_RInfo, Rtemp)); 1507 __ cmp(klass_RInfo, k_RInfo); 1508 __ cond_cmp(Rtemp, k_RInfo, ne); 1509 __ b(*success_target, eq); 1510 // check for immediate negative hit 1511 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1512 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset())); 1513 __ b(*failure_target, ne); 1514 // slow case 1515 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1516 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1517 __ cbz(R0, *failure_target); 1518 } 1519 1520 #else // AARCH64 1521 1522 __ movs(res, obj); 1523 if (op->should_profile()) { 1524 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done); 1525 } else { 1526 __ b(done, eq); 1527 } 1528 if (k->is_loaded()) { 1529 __ mov_metadata(k_RInfo, k->constant_encoding()); 1530 } else if (k_RInfo != obj) { 1531 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1532 __ movs(res, obj); 1533 } else { 1534 // Patching doesn't update "res" register after GC, so do patching first 1535 klass2reg_with_patching(Rtemp, op->info_for_patch()); 1536 __ movs(res, obj); 1537 __ mov(k_RInfo, Rtemp); 1538 } 1539 __ load_klass(klass_RInfo, res, ne); 1540 1541 if (op->fast_check()) { 1542 __ cmp(klass_RInfo, k_RInfo, ne); 1543 __ b(*failure_target, ne); 1544 } else if (k->is_loaded()) { 1545 __ b(*success_target, eq); 1546 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset())); 1547 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) { 1548 __ cmp(Rtemp, k_RInfo); 1549 __ b(*failure_target, ne); 1550 } else { 1551 __ cmp(klass_RInfo, k_RInfo); 1552 __ cmp(Rtemp, k_RInfo, ne); 1553 __ b(*success_target, eq); 1554 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1555 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1556 __ cbz(R0, *failure_target); 1557 } 1558 } else { 1559 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1560 __ b(*success_target, eq); 1561 // check for immediate positive hit 1562 __ ldr(Rtemp, Address(klass_RInfo, Rtemp)); 1563 __ cmp(klass_RInfo, k_RInfo); 1564 __ cmp(Rtemp, k_RInfo, ne); 1565 __ b(*success_target, eq); 1566 // check for immediate negative hit 1567 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1568 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset())); 1569 __ b(*failure_target, ne); 1570 // slow case 1571 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1572 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1573 __ cbz(R0, *failure_target); 1574 } 1575 #endif // AARCH64 1576 1577 if (op->should_profile()) { 1578 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp; 1579 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1, 1580 &profile_cast_success, &profile_cast_failure, 1581 &done, stub->entry()); 1582 } 1583 __ bind(done); 1584 break; 1585 } 1586 1587 case lir_instanceof: { 1588 Register obj = op->object()->as_register(); 1589 Register res = op->result_opr()->as_register(); 1590 Register klass_RInfo = op->tmp1()->as_register(); 1591 Register k_RInfo = op->tmp2()->as_register(); 1592 ciKlass* k = op->klass(); 1593 assert_different_registers(res, klass_RInfo, k_RInfo, Rtemp); 1594 1595 ciMethodData* md; 1596 ciProfileData* data; 1597 int mdo_offset_bias = 0; 1598 1599 Label done; 1600 1601 Label profile_cast_failure, profile_cast_success; 1602 Label *failure_target = op->should_profile() ? &profile_cast_failure : &done; 1603 Label *success_target = op->should_profile() ? &profile_cast_success : &done; 1604 1605 #ifdef AARCH64 1606 move_regs(obj, res); 1607 #else 1608 __ movs(res, obj); 1609 #endif // AARCH64 1610 1611 if (op->should_profile()) { 1612 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done); 1613 } else { 1614 #ifdef AARCH64 1615 __ cbz(obj, done); // If obj == NULL, res is false 1616 #else 1617 __ b(done, eq); 1618 #endif // AARCH64 1619 } 1620 1621 if (k->is_loaded()) { 1622 __ mov_metadata(k_RInfo, k->constant_encoding()); 1623 } else { 1624 op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res)); 1625 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1626 } 1627 __ load_klass(klass_RInfo, res); 1628 1629 #ifndef AARCH64 1630 if (!op->should_profile()) { 1631 __ mov(res, 0); 1632 } 1633 #endif // !AARCH64 1634 1635 if (op->fast_check()) { 1636 __ cmp(klass_RInfo, k_RInfo); 1637 if (!op->should_profile()) { 1638 set_instanceof_result(_masm, res, eq); 1639 } else { 1640 __ b(profile_cast_failure, ne); 1641 } 1642 } else if (k->is_loaded()) { 1643 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset())); 1644 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) { 1645 __ cmp(Rtemp, k_RInfo); 1646 if (!op->should_profile()) { 1647 set_instanceof_result(_masm, res, eq); 1648 } else { 1649 __ b(profile_cast_failure, ne); 1650 } 1651 } else { 1652 __ cmp(klass_RInfo, k_RInfo); 1653 __ cond_cmp(Rtemp, k_RInfo, ne); 1654 if (!op->should_profile()) { 1655 set_instanceof_result(_masm, res, eq); 1656 } 1657 __ b(*success_target, eq); 1658 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1659 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1660 if (!op->should_profile()) { 1661 move_regs(R0, res); 1662 } else { 1663 __ cbz(R0, *failure_target); 1664 } 1665 } 1666 } else { 1667 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1668 // check for immediate positive hit 1669 __ cmp(klass_RInfo, k_RInfo); 1670 if (!op->should_profile()) { 1671 #ifdef AARCH64 1672 // TODO-AARCH64 check if separate conditional branch is more efficient than ldr+cond_cmp 1673 __ ldr(res, Address(klass_RInfo, Rtemp)); 1674 #else 1675 __ ldr(res, Address(klass_RInfo, Rtemp), ne); 1676 #endif // AARCH64 1677 __ cond_cmp(res, k_RInfo, ne); 1678 set_instanceof_result(_masm, res, eq); 1679 } else { 1680 #ifdef AARCH64 1681 // TODO-AARCH64 check if separate conditional branch is more efficient than ldr+cond_cmp 1682 __ ldr(Rtemp, Address(klass_RInfo, Rtemp)); 1683 #else 1684 __ ldr(Rtemp, Address(klass_RInfo, Rtemp), ne); 1685 #endif // AARCH64 1686 __ cond_cmp(Rtemp, k_RInfo, ne); 1687 } 1688 __ b(*success_target, eq); 1689 // check for immediate negative hit 1690 if (op->should_profile()) { 1691 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset())); 1692 } 1693 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset())); 1694 if (!op->should_profile()) { 1695 #ifdef AARCH64 1696 __ mov(res, 0); 1697 #else 1698 __ mov(res, 0, ne); 1699 #endif // AARCH64 1700 } 1701 __ b(*failure_target, ne); 1702 // slow case 1703 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup"); 1704 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); 1705 if (!op->should_profile()) { 1706 move_regs(R0, res); 1707 } 1708 if (op->should_profile()) { 1709 __ cbz(R0, *failure_target); 1710 } 1711 } 1712 1713 if (op->should_profile()) { 1714 Label done_ok, done_failure; 1715 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp; 1716 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1, 1717 &profile_cast_success, &profile_cast_failure, 1718 &done_ok, &done_failure); 1719 __ bind(done_failure); 1720 __ mov(res, 0); 1721 __ b(done); 1722 __ bind(done_ok); 1723 __ mov(res, 1); 1724 } 1725 __ bind(done); 1726 break; 1727 } 1728 default: 1729 ShouldNotReachHere(); 1730 } 1731 } 1732 1733 1734 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1735 // if (*addr == cmpval) { 1736 // *addr = newval; 1737 // dest = 1; 1738 // } else { 1739 // dest = 0; 1740 // } 1741 #ifdef AARCH64 1742 Label retry, done; 1743 Register addr = op->addr()->as_pointer_register(); 1744 Register cmpval = op->cmp_value()->as_pointer_register(); 1745 Register newval = op->new_value()->as_pointer_register(); 1746 Register dest = op->result_opr()->as_pointer_register(); 1747 assert_different_registers(dest, addr, cmpval, newval, Rtemp); 1748 1749 if (UseCompressedOops && op->code() == lir_cas_obj) { 1750 Register tmp1 = op->tmp1()->as_pointer_register(); 1751 Register tmp2 = op->tmp2()->as_pointer_register(); 1752 assert_different_registers(dest, addr, cmpval, newval, tmp1, tmp2, Rtemp); 1753 __ encode_heap_oop(tmp1, cmpval); cmpval = tmp1; 1754 __ encode_heap_oop(tmp2, newval); newval = tmp2; 1755 } 1756 1757 __ mov(dest, ZR); 1758 __ bind(retry); 1759 if (((op->code() == lir_cas_obj) && !UseCompressedOops) || op->code() == lir_cas_long) { 1760 __ ldaxr(Rtemp, addr); 1761 __ cmp(Rtemp, cmpval); 1762 __ b(done, ne); 1763 __ stlxr(Rtemp, newval, addr); 1764 } else if (((op->code() == lir_cas_obj) && UseCompressedOops) || op->code() == lir_cas_int) { 1765 __ ldaxr_w(Rtemp, addr); 1766 __ cmp_w(Rtemp, cmpval); 1767 __ b(done, ne); 1768 __ stlxr_w(Rtemp, newval, addr); 1769 } else { 1770 ShouldNotReachHere(); 1771 } 1772 __ cbnz_w(Rtemp, retry); 1773 __ mov(dest, 1); 1774 __ bind(done); 1775 #else 1776 // FIXME: membar_release 1777 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp); 1778 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) { 1779 Register addr = op->addr()->as_register(); 1780 Register cmpval = op->cmp_value()->as_register(); 1781 Register newval = op->new_value()->as_register(); 1782 Register dest = op->result_opr()->as_register(); 1783 assert_different_registers(dest, addr, cmpval, newval, Rtemp); 1784 1785 __ atomic_cas_bool(cmpval, newval, addr, 0, Rtemp); // Rtemp free by default at C1 LIR layer 1786 __ mov(dest, 1, eq); 1787 __ mov(dest, 0, ne); 1788 } else if (op->code() == lir_cas_long) { 1789 assert(VM_Version::supports_cx8(), "wrong machine"); 1790 Register addr = op->addr()->as_pointer_register(); 1791 Register cmp_value_lo = op->cmp_value()->as_register_lo(); 1792 Register cmp_value_hi = op->cmp_value()->as_register_hi(); 1793 Register new_value_lo = op->new_value()->as_register_lo(); 1794 Register new_value_hi = op->new_value()->as_register_hi(); 1795 Register dest = op->result_opr()->as_register(); 1796 Register tmp_lo = op->tmp1()->as_register_lo(); 1797 Register tmp_hi = op->tmp1()->as_register_hi(); 1798 1799 assert_different_registers(tmp_lo, tmp_hi, cmp_value_lo, cmp_value_hi, dest, new_value_lo, new_value_hi, addr); 1800 assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair"); 1801 assert(new_value_hi->encoding() == new_value_lo->encoding() + 1, "non aligned register pair"); 1802 assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair"); 1803 assert((new_value_lo->encoding() & 0x1) == 0, "misaligned register pair"); 1804 __ atomic_cas64(tmp_lo, tmp_hi, dest, cmp_value_lo, cmp_value_hi, 1805 new_value_lo, new_value_hi, addr, 0); 1806 } else { 1807 Unimplemented(); 1808 } 1809 #endif // AARCH64 1810 // FIXME: is full membar really needed instead of just membar_acquire? 1811 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp); 1812 } 1813 1814 1815 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) { 1816 AsmCondition acond = al; 1817 AsmCondition ncond = nv; 1818 if (opr1 != opr2) { 1819 switch (condition) { 1820 case lir_cond_equal: acond = eq; ncond = ne; break; 1821 case lir_cond_notEqual: acond = ne; ncond = eq; break; 1822 case lir_cond_less: acond = lt; ncond = ge; break; 1823 case lir_cond_lessEqual: acond = le; ncond = gt; break; 1824 case lir_cond_greaterEqual: acond = ge; ncond = lt; break; 1825 case lir_cond_greater: acond = gt; ncond = le; break; 1826 case lir_cond_aboveEqual: acond = hs; ncond = lo; break; 1827 case lir_cond_belowEqual: acond = ls; ncond = hi; break; 1828 default: ShouldNotReachHere(); 1829 } 1830 } 1831 1832 #ifdef AARCH64 1833 1834 // TODO-AARCH64 implement it more efficiently 1835 1836 if (opr1->is_register()) { 1837 reg2reg(opr1, result); 1838 } else if (opr1->is_stack()) { 1839 stack2reg(opr1, result, result->type()); 1840 } else if (opr1->is_constant()) { 1841 const2reg(opr1, result, lir_patch_none, NULL); 1842 } else { 1843 ShouldNotReachHere(); 1844 } 1845 1846 Label skip; 1847 __ b(skip, acond); 1848 1849 if (opr2->is_register()) { 1850 reg2reg(opr2, result); 1851 } else if (opr2->is_stack()) { 1852 stack2reg(opr2, result, result->type()); 1853 } else if (opr2->is_constant()) { 1854 const2reg(opr2, result, lir_patch_none, NULL); 1855 } else { 1856 ShouldNotReachHere(); 1857 } 1858 1859 __ bind(skip); 1860 1861 #else 1862 for (;;) { // two iterations only 1863 if (opr1 == result) { 1864 // do nothing 1865 } else if (opr1->is_single_cpu()) { 1866 __ mov(result->as_register(), opr1->as_register(), acond); 1867 } else if (opr1->is_double_cpu()) { 1868 __ long_move(result->as_register_lo(), result->as_register_hi(), 1869 opr1->as_register_lo(), opr1->as_register_hi(), acond); 1870 } else if (opr1->is_single_stack()) { 1871 __ ldr(result->as_register(), frame_map()->address_for_slot(opr1->single_stack_ix()), acond); 1872 } else if (opr1->is_double_stack()) { 1873 __ ldr(result->as_register_lo(), 1874 frame_map()->address_for_slot(opr1->double_stack_ix(), lo_word_offset_in_bytes), acond); 1875 __ ldr(result->as_register_hi(), 1876 frame_map()->address_for_slot(opr1->double_stack_ix(), hi_word_offset_in_bytes), acond); 1877 } else if (opr1->is_illegal()) { 1878 // do nothing: this part of the cmove has been optimized away in the peephole optimizer 1879 } else { 1880 assert(opr1->is_constant(), "must be"); 1881 LIR_Const* c = opr1->as_constant_ptr(); 1882 1883 switch (c->type()) { 1884 case T_INT: 1885 __ mov_slow(result->as_register(), c->as_jint(), acond); 1886 break; 1887 case T_LONG: 1888 __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond); 1889 __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond); 1890 break; 1891 case T_OBJECT: 1892 __ mov_oop(result->as_register(), c->as_jobject(), 0, acond); 1893 break; 1894 case T_FLOAT: 1895 #ifdef __SOFTFP__ 1896 // not generated now. 1897 __ mov_slow(result->as_register(), c->as_jint(), acond); 1898 #else 1899 __ mov_float(result->as_float_reg(), c->as_jfloat(), acond); 1900 #endif // __SOFTFP__ 1901 break; 1902 case T_DOUBLE: 1903 #ifdef __SOFTFP__ 1904 // not generated now. 1905 __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond); 1906 __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond); 1907 #else 1908 __ mov_double(result->as_double_reg(), c->as_jdouble(), acond); 1909 #endif // __SOFTFP__ 1910 break; 1911 default: 1912 ShouldNotReachHere(); 1913 } 1914 } 1915 1916 // Negate the condition and repeat the algorithm with the second operand 1917 if (opr1 == opr2) { break; } 1918 opr1 = opr2; 1919 acond = ncond; 1920 } 1921 #endif // AARCH64 1922 } 1923 1924 #if defined(AARCH64) || defined(ASSERT) 1925 static int reg_size(LIR_Opr op) { 1926 switch (op->type()) { 1927 case T_FLOAT: 1928 case T_INT: return BytesPerInt; 1929 case T_LONG: 1930 case T_DOUBLE: return BytesPerLong; 1931 case T_OBJECT: 1932 case T_ARRAY: 1933 case T_METADATA: return BytesPerWord; 1934 case T_ADDRESS: 1935 case T_ILLEGAL: // fall through 1936 default: ShouldNotReachHere(); return -1; 1937 } 1938 } 1939 #endif 1940 1941 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 1942 assert(info == NULL, "unused on this code path"); 1943 assert(dest->is_register(), "wrong items state"); 1944 1945 if (right->is_address()) { 1946 // special case for adding shifted/extended register 1947 const Register res = dest->as_pointer_register(); 1948 const Register lreg = left->as_pointer_register(); 1949 const LIR_Address* addr = right->as_address_ptr(); 1950 1951 assert(addr->base()->as_pointer_register() == lreg && addr->index()->is_register() && addr->disp() == 0, "must be"); 1952 1953 int scale = addr->scale(); 1954 AsmShift shift = lsl; 1955 1956 #ifdef AARCH64 1957 bool is_index_extended = reg_size(addr->base()) > reg_size(addr->index()); 1958 if (scale < 0) { 1959 scale = -scale; 1960 shift = lsr; 1961 } 1962 assert(shift == lsl || !is_index_extended, "could not have extend and right shift in one operand"); 1963 assert(0 <= scale && scale <= 63, "scale is too large"); 1964 1965 if (is_index_extended) { 1966 assert(scale <= 4, "scale is too large for add with extended register"); 1967 assert(addr->index()->is_single_cpu(), "should be"); 1968 assert(addr->index()->type() == T_INT, "should be"); 1969 assert(dest->is_double_cpu(), "should be"); 1970 assert(code == lir_add, "special case of add with extended register"); 1971 1972 __ add(res, lreg, addr->index()->as_register(), ex_sxtw, scale); 1973 return; 1974 } else if (reg_size(dest) == BytesPerInt) { 1975 assert(reg_size(addr->base()) == reg_size(addr->index()), "should be"); 1976 assert(reg_size(addr->base()) == reg_size(dest), "should be"); 1977 1978 AsmOperand operand(addr->index()->as_pointer_register(), shift, scale); 1979 switch (code) { 1980 case lir_add: __ add_32(res, lreg, operand); break; 1981 case lir_sub: __ sub_32(res, lreg, operand); break; 1982 default: ShouldNotReachHere(); 1983 } 1984 return; 1985 } 1986 #endif // AARCH64 1987 1988 assert(reg_size(addr->base()) == reg_size(addr->index()), "should be"); 1989 assert(reg_size(addr->base()) == reg_size(dest), "should be"); 1990 assert(reg_size(dest) == wordSize, "should be"); 1991 1992 AsmOperand operand(addr->index()->as_pointer_register(), shift, scale); 1993 switch (code) { 1994 case lir_add: __ add(res, lreg, operand); break; 1995 case lir_sub: __ sub(res, lreg, operand); break; 1996 default: ShouldNotReachHere(); 1997 } 1998 1999 #ifndef AARCH64 2000 } else if (left->is_address()) { 2001 assert(code == lir_sub && right->is_single_cpu(), "special case used by strength_reduce_multiply()"); 2002 const LIR_Address* addr = left->as_address_ptr(); 2003 const Register res = dest->as_register(); 2004 const Register rreg = right->as_register(); 2005 assert(addr->base()->as_register() == rreg && addr->index()->is_register() && addr->disp() == 0, "must be"); 2006 __ rsb(res, rreg, AsmOperand(addr->index()->as_register(), lsl, addr->scale())); 2007 #endif // !AARCH64 2008 2009 } else if (dest->is_single_cpu()) { 2010 assert(left->is_single_cpu(), "unexpected left operand"); 2011 #ifdef AARCH64 2012 assert(dest->type() == T_INT, "unexpected dest type"); 2013 assert(left->type() == T_INT, "unexpected left type"); 2014 assert(right->type() == T_INT, "unexpected right type"); 2015 #endif // AARCH64 2016 2017 const Register res = dest->as_register(); 2018 const Register lreg = left->as_register(); 2019 2020 if (right->is_single_cpu()) { 2021 const Register rreg = right->as_register(); 2022 switch (code) { 2023 case lir_add: __ add_32(res, lreg, rreg); break; 2024 case lir_sub: __ sub_32(res, lreg, rreg); break; 2025 case lir_mul: __ mul_32(res, lreg, rreg); break; 2026 default: ShouldNotReachHere(); 2027 } 2028 } else { 2029 assert(right->is_constant(), "must be"); 2030 const jint c = right->as_constant_ptr()->as_jint(); 2031 if (!Assembler::is_arith_imm_in_range(c)) { 2032 BAILOUT("illegal arithmetic operand"); 2033 } 2034 switch (code) { 2035 case lir_add: __ add_32(res, lreg, c); break; 2036 case lir_sub: __ sub_32(res, lreg, c); break; 2037 default: ShouldNotReachHere(); 2038 } 2039 } 2040 2041 } else if (dest->is_double_cpu()) { 2042 #ifdef AARCH64 2043 assert(left->is_double_cpu() || 2044 (left->is_single_cpu() && ((left->type() == T_OBJECT) || (left->type() == T_ARRAY) || (left->type() == T_ADDRESS))), 2045 "unexpected left operand"); 2046 2047 const Register res = dest->as_register_lo(); 2048 const Register lreg = left->as_pointer_register(); 2049 2050 if (right->is_constant()) { 2051 assert(right->type() == T_LONG, "unexpected right type"); 2052 assert((right->as_constant_ptr()->as_jlong() >> 24) == 0, "out of range"); 2053 jint imm = (jint)right->as_constant_ptr()->as_jlong(); 2054 switch (code) { 2055 case lir_add: __ add(res, lreg, imm); break; 2056 case lir_sub: __ sub(res, lreg, imm); break; 2057 default: ShouldNotReachHere(); 2058 } 2059 } else { 2060 assert(right->is_double_cpu() || 2061 (right->is_single_cpu() && ((right->type() == T_OBJECT) || (right->type() == T_ARRAY) || (right->type() == T_ADDRESS))), 2062 "unexpected right operand"); 2063 const Register rreg = right->as_pointer_register(); 2064 switch (code) { 2065 case lir_add: __ add(res, lreg, rreg); break; 2066 case lir_sub: __ sub(res, lreg, rreg); break; 2067 case lir_mul: __ mul(res, lreg, rreg); break; 2068 default: ShouldNotReachHere(); 2069 } 2070 } 2071 #else // AARCH64 2072 Register res_lo = dest->as_register_lo(); 2073 Register res_hi = dest->as_register_hi(); 2074 Register lreg_lo = left->as_register_lo(); 2075 Register lreg_hi = left->as_register_hi(); 2076 if (right->is_double_cpu()) { 2077 Register rreg_lo = right->as_register_lo(); 2078 Register rreg_hi = right->as_register_hi(); 2079 if (res_lo == lreg_hi || res_lo == rreg_hi) { 2080 res_lo = Rtemp; 2081 } 2082 switch (code) { 2083 case lir_add: 2084 __ adds(res_lo, lreg_lo, rreg_lo); 2085 __ adc(res_hi, lreg_hi, rreg_hi); 2086 break; 2087 case lir_sub: 2088 __ subs(res_lo, lreg_lo, rreg_lo); 2089 __ sbc(res_hi, lreg_hi, rreg_hi); 2090 break; 2091 default: 2092 ShouldNotReachHere(); 2093 } 2094 } else { 2095 assert(right->is_constant(), "must be"); 2096 assert((right->as_constant_ptr()->as_jlong() >> 32) == 0, "out of range"); 2097 const jint c = (jint) right->as_constant_ptr()->as_jlong(); 2098 if (res_lo == lreg_hi) { 2099 res_lo = Rtemp; 2100 } 2101 switch (code) { 2102 case lir_add: 2103 __ adds(res_lo, lreg_lo, c); 2104 __ adc(res_hi, lreg_hi, 0); 2105 break; 2106 case lir_sub: 2107 __ subs(res_lo, lreg_lo, c); 2108 __ sbc(res_hi, lreg_hi, 0); 2109 break; 2110 default: 2111 ShouldNotReachHere(); 2112 } 2113 } 2114 move_regs(res_lo, dest->as_register_lo()); 2115 #endif // AARCH64 2116 2117 } else if (dest->is_single_fpu()) { 2118 assert(left->is_single_fpu(), "must be"); 2119 assert(right->is_single_fpu(), "must be"); 2120 const FloatRegister res = dest->as_float_reg(); 2121 const FloatRegister lreg = left->as_float_reg(); 2122 const FloatRegister rreg = right->as_float_reg(); 2123 switch (code) { 2124 case lir_add: __ add_float(res, lreg, rreg); break; 2125 case lir_sub: __ sub_float(res, lreg, rreg); break; 2126 case lir_mul_strictfp: // fall through 2127 case lir_mul: __ mul_float(res, lreg, rreg); break; 2128 case lir_div_strictfp: // fall through 2129 case lir_div: __ div_float(res, lreg, rreg); break; 2130 default: ShouldNotReachHere(); 2131 } 2132 } else if (dest->is_double_fpu()) { 2133 assert(left->is_double_fpu(), "must be"); 2134 assert(right->is_double_fpu(), "must be"); 2135 const FloatRegister res = dest->as_double_reg(); 2136 const FloatRegister lreg = left->as_double_reg(); 2137 const FloatRegister rreg = right->as_double_reg(); 2138 switch (code) { 2139 case lir_add: __ add_double(res, lreg, rreg); break; 2140 case lir_sub: __ sub_double(res, lreg, rreg); break; 2141 case lir_mul_strictfp: // fall through 2142 case lir_mul: __ mul_double(res, lreg, rreg); break; 2143 case lir_div_strictfp: // fall through 2144 case lir_div: __ div_double(res, lreg, rreg); break; 2145 default: ShouldNotReachHere(); 2146 } 2147 } else { 2148 ShouldNotReachHere(); 2149 } 2150 } 2151 2152 2153 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { 2154 switch (code) { 2155 case lir_abs: 2156 __ abs_double(dest->as_double_reg(), value->as_double_reg()); 2157 break; 2158 case lir_sqrt: 2159 __ sqrt_double(dest->as_double_reg(), value->as_double_reg()); 2160 break; 2161 default: 2162 ShouldNotReachHere(); 2163 } 2164 } 2165 2166 2167 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) { 2168 assert(dest->is_register(), "wrong items state"); 2169 assert(left->is_register(), "wrong items state"); 2170 2171 if (dest->is_single_cpu()) { 2172 #ifdef AARCH64 2173 assert (dest->type() == T_INT, "unexpected result type"); 2174 assert (left->type() == T_INT, "unexpected left type"); 2175 assert (right->type() == T_INT, "unexpected right type"); 2176 #endif // AARCH64 2177 2178 const Register res = dest->as_register(); 2179 const Register lreg = left->as_register(); 2180 2181 if (right->is_single_cpu()) { 2182 const Register rreg = right->as_register(); 2183 switch (code) { 2184 case lir_logic_and: __ and_32(res, lreg, rreg); break; 2185 case lir_logic_or: __ orr_32(res, lreg, rreg); break; 2186 case lir_logic_xor: __ eor_32(res, lreg, rreg); break; 2187 default: ShouldNotReachHere(); 2188 } 2189 } else { 2190 assert(right->is_constant(), "must be"); 2191 const uint c = (uint)right->as_constant_ptr()->as_jint(); 2192 switch (code) { 2193 case lir_logic_and: __ and_32(res, lreg, c); break; 2194 case lir_logic_or: __ orr_32(res, lreg, c); break; 2195 case lir_logic_xor: __ eor_32(res, lreg, c); break; 2196 default: ShouldNotReachHere(); 2197 } 2198 } 2199 } else { 2200 assert(dest->is_double_cpu(), "should be"); 2201 Register res_lo = dest->as_register_lo(); 2202 2203 #ifdef AARCH64 2204 assert ((left->is_single_cpu() && left->is_oop_register()) || left->is_double_cpu(), "should be"); 2205 const Register lreg_lo = left->as_pointer_register(); 2206 #else 2207 assert (dest->type() == T_LONG, "unexpected result type"); 2208 assert (left->type() == T_LONG, "unexpected left type"); 2209 assert (right->type() == T_LONG, "unexpected right type"); 2210 2211 const Register res_hi = dest->as_register_hi(); 2212 const Register lreg_lo = left->as_register_lo(); 2213 const Register lreg_hi = left->as_register_hi(); 2214 #endif // AARCH64 2215 2216 if (right->is_register()) { 2217 #ifdef AARCH64 2218 assert ((right->is_single_cpu() && right->is_oop_register()) || right->is_double_cpu(), "should be"); 2219 const Register rreg_lo = right->as_pointer_register(); 2220 switch (code) { 2221 case lir_logic_and: __ andr(res_lo, lreg_lo, rreg_lo); break; 2222 case lir_logic_or: __ orr (res_lo, lreg_lo, rreg_lo); break; 2223 case lir_logic_xor: __ eor (res_lo, lreg_lo, rreg_lo); break; 2224 default: ShouldNotReachHere(); 2225 } 2226 #else 2227 const Register rreg_lo = right->as_register_lo(); 2228 const Register rreg_hi = right->as_register_hi(); 2229 if (res_lo == lreg_hi || res_lo == rreg_hi) { 2230 res_lo = Rtemp; // Temp register helps to avoid overlap between result and input 2231 } 2232 switch (code) { 2233 case lir_logic_and: 2234 __ andr(res_lo, lreg_lo, rreg_lo); 2235 __ andr(res_hi, lreg_hi, rreg_hi); 2236 break; 2237 case lir_logic_or: 2238 __ orr(res_lo, lreg_lo, rreg_lo); 2239 __ orr(res_hi, lreg_hi, rreg_hi); 2240 break; 2241 case lir_logic_xor: 2242 __ eor(res_lo, lreg_lo, rreg_lo); 2243 __ eor(res_hi, lreg_hi, rreg_hi); 2244 break; 2245 default: 2246 ShouldNotReachHere(); 2247 } 2248 move_regs(res_lo, dest->as_register_lo()); 2249 #endif // AARCH64 2250 } else { 2251 assert(right->is_constant(), "must be"); 2252 #ifdef AARCH64 2253 const julong c = (julong)right->as_constant_ptr()->as_jlong(); 2254 Assembler::LogicalImmediate imm(c, false); 2255 if (imm.is_encoded()) { 2256 switch (code) { 2257 case lir_logic_and: __ andr(res_lo, lreg_lo, imm); break; 2258 case lir_logic_or: __ orr (res_lo, lreg_lo, imm); break; 2259 case lir_logic_xor: __ eor (res_lo, lreg_lo, imm); break; 2260 default: ShouldNotReachHere(); 2261 } 2262 } else { 2263 BAILOUT("64 bit constant cannot be inlined"); 2264 } 2265 #else 2266 const jint c_lo = (jint) right->as_constant_ptr()->as_jlong(); 2267 const jint c_hi = (jint) (right->as_constant_ptr()->as_jlong() >> 32); 2268 // Case for logic_or from do_ClassIDIntrinsic() 2269 if (c_hi == 0 && AsmOperand::is_rotated_imm(c_lo)) { 2270 switch (code) { 2271 case lir_logic_and: 2272 __ andr(res_lo, lreg_lo, c_lo); 2273 __ mov(res_hi, 0); 2274 break; 2275 case lir_logic_or: 2276 __ orr(res_lo, lreg_lo, c_lo); 2277 break; 2278 case lir_logic_xor: 2279 __ eor(res_lo, lreg_lo, c_lo); 2280 break; 2281 default: 2282 ShouldNotReachHere(); 2283 } 2284 } else if (code == lir_logic_and && 2285 c_hi == -1 && 2286 (AsmOperand::is_rotated_imm(c_lo) || 2287 AsmOperand::is_rotated_imm(~c_lo))) { 2288 // Another case which handles logic_and from do_ClassIDIntrinsic() 2289 if (AsmOperand::is_rotated_imm(c_lo)) { 2290 __ andr(res_lo, lreg_lo, c_lo); 2291 } else { 2292 __ bic(res_lo, lreg_lo, ~c_lo); 2293 } 2294 if (res_hi != lreg_hi) { 2295 __ mov(res_hi, lreg_hi); 2296 } 2297 } else { 2298 BAILOUT("64 bit constant cannot be inlined"); 2299 } 2300 #endif // AARCH64 2301 } 2302 } 2303 } 2304 2305 2306 #ifdef AARCH64 2307 2308 void LIR_Assembler::long_compare_helper(LIR_Opr opr1, LIR_Opr opr2) { 2309 assert(opr1->is_double_cpu(), "should be"); 2310 Register x = opr1->as_register_lo(); 2311 2312 if (opr2->is_double_cpu()) { 2313 Register y = opr2->as_register_lo(); 2314 __ cmp(x, y); 2315 2316 } else { 2317 assert(opr2->is_constant(), "should be"); 2318 assert(opr2->as_constant_ptr()->type() == T_LONG, "long constant expected"); 2319 jlong c = opr2->as_jlong(); 2320 assert(((c >> 31) == 0) || ((c >> 31) == -1), "immediate is out of range"); 2321 if (c >= 0) { 2322 __ cmp(x, (jint)c); 2323 } else { 2324 __ cmn(x, (jint)(-c)); 2325 } 2326 } 2327 } 2328 2329 #endif // AARCH64 2330 2331 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 2332 if (opr1->is_single_cpu()) { 2333 if (opr2->is_constant()) { 2334 switch (opr2->as_constant_ptr()->type()) { 2335 case T_INT: { 2336 const jint c = opr2->as_constant_ptr()->as_jint(); 2337 if (Assembler::is_arith_imm_in_range(c)) { 2338 __ cmp_32(opr1->as_register(), c); 2339 } else if (Assembler::is_arith_imm_in_range(-c)) { 2340 __ cmn_32(opr1->as_register(), -c); 2341 } else { 2342 // This can happen when compiling lookupswitch 2343 __ mov_slow(Rtemp, c); 2344 __ cmp_32(opr1->as_register(), Rtemp); 2345 } 2346 break; 2347 } 2348 case T_OBJECT: 2349 assert(opr2->as_constant_ptr()->as_jobject() == NULL, "cannot handle otherwise"); 2350 __ cmp(opr1->as_register(), 0); 2351 break; 2352 default: 2353 ShouldNotReachHere(); 2354 } 2355 } else if (opr2->is_single_cpu()) { 2356 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_METADATA || opr1->type() == T_ADDRESS) { 2357 assert(opr2->type() == T_OBJECT || opr2->type() == T_ARRAY || opr2->type() == T_METADATA || opr2->type() == T_ADDRESS, "incompatibe type"); 2358 __ cmp(opr1->as_register(), opr2->as_register()); 2359 } else { 2360 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY && opr2->type() != T_METADATA && opr2->type() != T_ADDRESS, "incompatibe type"); 2361 __ cmp_32(opr1->as_register(), opr2->as_register()); 2362 } 2363 } else { 2364 ShouldNotReachHere(); 2365 } 2366 } else if (opr1->is_double_cpu()) { 2367 #ifdef AARCH64 2368 long_compare_helper(opr1, opr2); 2369 #else 2370 Register xlo = opr1->as_register_lo(); 2371 Register xhi = opr1->as_register_hi(); 2372 if (opr2->is_constant() && opr2->as_jlong() == 0) { 2373 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "cannot handle otherwise"); 2374 __ orrs(Rtemp, xlo, xhi); 2375 } else if (opr2->is_register()) { 2376 Register ylo = opr2->as_register_lo(); 2377 Register yhi = opr2->as_register_hi(); 2378 if (condition == lir_cond_equal || condition == lir_cond_notEqual) { 2379 __ teq(xhi, yhi); 2380 __ teq(xlo, ylo, eq); 2381 } else { 2382 __ subs(xlo, xlo, ylo); 2383 __ sbcs(xhi, xhi, yhi); 2384 } 2385 } else { 2386 ShouldNotReachHere(); 2387 } 2388 #endif // AARCH64 2389 } else if (opr1->is_single_fpu()) { 2390 if (opr2->is_constant()) { 2391 assert(opr2->as_jfloat() == 0.0f, "cannot handle otherwise"); 2392 __ cmp_zero_float(opr1->as_float_reg()); 2393 } else { 2394 __ cmp_float(opr1->as_float_reg(), opr2->as_float_reg()); 2395 } 2396 } else if (opr1->is_double_fpu()) { 2397 if (opr2->is_constant()) { 2398 assert(opr2->as_jdouble() == 0.0, "cannot handle otherwise"); 2399 __ cmp_zero_double(opr1->as_double_reg()); 2400 } else { 2401 __ cmp_double(opr1->as_double_reg(), opr2->as_double_reg()); 2402 } 2403 } else { 2404 ShouldNotReachHere(); 2405 } 2406 } 2407 2408 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) { 2409 const Register res = dst->as_register(); 2410 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 2411 comp_op(lir_cond_unknown, left, right, op); 2412 #ifdef AARCH64 2413 if (code == lir_ucmp_fd2i) { // unordered is less 2414 __ cset(res, gt); // 1 if '>', else 0 2415 __ csinv(res, res, ZR, ge); // previous value if '>=', else -1 2416 } else { 2417 __ cset(res, hi); // 1 if '>' or unordered, else 0 2418 __ csinv(res, res, ZR, pl); // previous value if '>=' or unordered, else -1 2419 } 2420 #else 2421 __ fmstat(); 2422 if (code == lir_ucmp_fd2i) { // unordered is less 2423 __ mvn(res, 0, lt); 2424 __ mov(res, 1, ge); 2425 } else { // unordered is greater 2426 __ mov(res, 1, cs); 2427 __ mvn(res, 0, cc); 2428 } 2429 __ mov(res, 0, eq); 2430 #endif // AARCH64 2431 2432 } else { 2433 assert(code == lir_cmp_l2i, "must be"); 2434 2435 #ifdef AARCH64 2436 long_compare_helper(left, right); 2437 2438 __ cset(res, gt); // 1 if '>', else 0 2439 __ csinv(res, res, ZR, ge); // previous value if '>=', else -1 2440 #else 2441 Label done; 2442 const Register xlo = left->as_register_lo(); 2443 const Register xhi = left->as_register_hi(); 2444 const Register ylo = right->as_register_lo(); 2445 const Register yhi = right->as_register_hi(); 2446 __ cmp(xhi, yhi); 2447 __ mov(res, 1, gt); 2448 __ mvn(res, 0, lt); 2449 __ b(done, ne); 2450 __ subs(res, xlo, ylo); 2451 __ mov(res, 1, hi); 2452 __ mvn(res, 0, lo); 2453 __ bind(done); 2454 #endif // AARCH64 2455 } 2456 } 2457 2458 2459 void LIR_Assembler::align_call(LIR_Code code) { 2460 // Not needed 2461 } 2462 2463 2464 void LIR_Assembler::call(LIR_OpJavaCall *op, relocInfo::relocType rtype) { 2465 int ret_addr_offset = __ patchable_call(op->addr(), rtype); 2466 assert(ret_addr_offset == __ offset(), "embedded return address not allowed"); 2467 add_call_info_here(op->info()); 2468 } 2469 2470 2471 void LIR_Assembler::ic_call(LIR_OpJavaCall *op) { 2472 bool near_range = __ cache_fully_reachable(); 2473 address oop_address = pc(); 2474 2475 bool use_movw = AARCH64_ONLY(false) NOT_AARCH64(VM_Version::supports_movw()); 2476 2477 // Ricklass may contain something that is not a metadata pointer so 2478 // mov_metadata can't be used 2479 InlinedAddress value((address)Universe::non_oop_word()); 2480 InlinedAddress addr(op->addr()); 2481 if (use_movw) { 2482 #ifdef AARCH64 2483 ShouldNotReachHere(); 2484 #else 2485 __ movw(Ricklass, ((unsigned int)Universe::non_oop_word()) & 0xffff); 2486 __ movt(Ricklass, ((unsigned int)Universe::non_oop_word()) >> 16); 2487 #endif // AARCH64 2488 } else { 2489 // No movw/movt, must be load a pc relative value but no 2490 // relocation so no metadata table to load from. 2491 // Use a b instruction rather than a bl, inline constant after the 2492 // branch, use a PC relative ldr to load the constant, arrange for 2493 // the call to return after the constant(s). 2494 __ ldr_literal(Ricklass, value); 2495 } 2496 __ relocate(virtual_call_Relocation::spec(oop_address)); 2497 if (near_range && use_movw) { 2498 __ bl(op->addr()); 2499 } else { 2500 Label call_return; 2501 __ adr(LR, call_return); 2502 if (near_range) { 2503 __ b(op->addr()); 2504 } else { 2505 __ indirect_jump(addr, Rtemp); 2506 __ bind_literal(addr); 2507 } 2508 if (!use_movw) { 2509 __ bind_literal(value); 2510 } 2511 __ bind(call_return); 2512 } 2513 add_call_info(code_offset(), op->info()); 2514 } 2515 2516 2517 /* Currently, vtable-dispatch is only enabled for sparc platforms */ 2518 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) { 2519 ShouldNotReachHere(); 2520 } 2521 2522 void LIR_Assembler::emit_static_call_stub() { 2523 address call_pc = __ pc(); 2524 address stub = __ start_a_stub(call_stub_size()); 2525 if (stub == NULL) { 2526 BAILOUT("static call stub overflow"); 2527 } 2528 2529 DEBUG_ONLY(int offset = code_offset();) 2530 2531 InlinedMetadata metadata_literal(NULL); 2532 __ relocate(static_stub_Relocation::spec(call_pc)); 2533 // If not a single instruction, NativeMovConstReg::next_instruction_address() 2534 // must jump over the whole following ldr_literal. 2535 // (See CompiledStaticCall::set_to_interpreted()) 2536 #ifdef ASSERT 2537 address ldr_site = __ pc(); 2538 #endif 2539 __ ldr_literal(Rmethod, metadata_literal); 2540 assert(nativeMovConstReg_at(ldr_site)->next_instruction_address() == __ pc(), "Fix ldr_literal or its parsing"); 2541 bool near_range = __ cache_fully_reachable(); 2542 InlinedAddress dest((address)-1); 2543 if (near_range) { 2544 address branch_site = __ pc(); 2545 __ b(branch_site); // b to self maps to special NativeJump -1 destination 2546 } else { 2547 __ indirect_jump(dest, Rtemp); 2548 } 2549 __ bind_literal(metadata_literal); // includes spec_for_immediate reloc 2550 if (!near_range) { 2551 __ bind_literal(dest); // special NativeJump -1 destination 2552 } 2553 2554 assert(code_offset() - offset <= call_stub_size(), "overflow"); 2555 __ end_a_stub(); 2556 } 2557 2558 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 2559 assert(exceptionOop->as_register() == Rexception_obj, "must match"); 2560 assert(exceptionPC->as_register() == Rexception_pc, "must match"); 2561 info->add_register_oop(exceptionOop); 2562 2563 Runtime1::StubID handle_id = compilation()->has_fpu_code() ? 2564 Runtime1::handle_exception_id : 2565 Runtime1::handle_exception_nofpu_id; 2566 Label return_address; 2567 __ adr(Rexception_pc, return_address); 2568 __ call(Runtime1::entry_for(handle_id), relocInfo::runtime_call_type); 2569 __ bind(return_address); 2570 add_call_info_here(info); // for exception handler 2571 } 2572 2573 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 2574 assert(exceptionOop->as_register() == Rexception_obj, "must match"); 2575 __ b(_unwind_handler_entry); 2576 } 2577 2578 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 2579 #ifdef AARCH64 2580 if (dest->is_single_cpu()) { 2581 Register res = dest->as_register(); 2582 Register x = left->as_register(); 2583 Register y = count->as_register(); 2584 assert (dest->type() == T_INT, "unexpected result type"); 2585 assert (left->type() == T_INT, "unexpected left type"); 2586 2587 switch (code) { 2588 case lir_shl: __ lslv_w(res, x, y); break; 2589 case lir_shr: __ asrv_w(res, x, y); break; 2590 case lir_ushr: __ lsrv_w(res, x, y); break; 2591 default: ShouldNotReachHere(); 2592 } 2593 } else if (dest->is_double_cpu()) { 2594 Register res = dest->as_register_lo(); 2595 Register x = left->as_register_lo(); 2596 Register y = count->as_register(); 2597 2598 switch (code) { 2599 case lir_shl: __ lslv(res, x, y); break; 2600 case lir_shr: __ asrv(res, x, y); break; 2601 case lir_ushr: __ lsrv(res, x, y); break; 2602 default: ShouldNotReachHere(); 2603 } 2604 } else { 2605 ShouldNotReachHere(); 2606 } 2607 #else 2608 AsmShift shift = lsl; 2609 switch (code) { 2610 case lir_shl: shift = lsl; break; 2611 case lir_shr: shift = asr; break; 2612 case lir_ushr: shift = lsr; break; 2613 default: ShouldNotReachHere(); 2614 } 2615 2616 if (dest->is_single_cpu()) { 2617 __ andr(Rtemp, count->as_register(), 31); 2618 __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, Rtemp)); 2619 } else if (dest->is_double_cpu()) { 2620 Register dest_lo = dest->as_register_lo(); 2621 Register dest_hi = dest->as_register_hi(); 2622 Register src_lo = left->as_register_lo(); 2623 Register src_hi = left->as_register_hi(); 2624 Register Rcount = count->as_register(); 2625 // Resolve possible register conflicts 2626 if (shift == lsl && dest_hi == src_lo) { 2627 dest_hi = Rtemp; 2628 } else if (shift != lsl && dest_lo == src_hi) { 2629 dest_lo = Rtemp; 2630 } else if (dest_lo == src_lo && dest_hi == src_hi) { 2631 dest_lo = Rtemp; 2632 } else if (dest_lo == Rcount || dest_hi == Rcount) { 2633 Rcount = Rtemp; 2634 } 2635 __ andr(Rcount, count->as_register(), 63); 2636 __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, Rcount); 2637 move_regs(dest_lo, dest->as_register_lo()); 2638 move_regs(dest_hi, dest->as_register_hi()); 2639 } else { 2640 ShouldNotReachHere(); 2641 } 2642 #endif // AARCH64 2643 } 2644 2645 2646 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 2647 #ifdef AARCH64 2648 if (dest->is_single_cpu()) { 2649 assert (dest->type() == T_INT, "unexpected result type"); 2650 assert (left->type() == T_INT, "unexpected left type"); 2651 count &= 31; 2652 if (count != 0) { 2653 switch (code) { 2654 case lir_shl: __ _lsl_w(dest->as_register(), left->as_register(), count); break; 2655 case lir_shr: __ _asr_w(dest->as_register(), left->as_register(), count); break; 2656 case lir_ushr: __ _lsr_w(dest->as_register(), left->as_register(), count); break; 2657 default: ShouldNotReachHere(); 2658 } 2659 } else { 2660 move_regs(left->as_register(), dest->as_register()); 2661 } 2662 } else if (dest->is_double_cpu()) { 2663 count &= 63; 2664 if (count != 0) { 2665 switch (code) { 2666 case lir_shl: __ _lsl(dest->as_register_lo(), left->as_register_lo(), count); break; 2667 case lir_shr: __ _asr(dest->as_register_lo(), left->as_register_lo(), count); break; 2668 case lir_ushr: __ _lsr(dest->as_register_lo(), left->as_register_lo(), count); break; 2669 default: ShouldNotReachHere(); 2670 } 2671 } else { 2672 move_regs(left->as_register_lo(), dest->as_register_lo()); 2673 } 2674 } else { 2675 ShouldNotReachHere(); 2676 } 2677 2678 #else 2679 AsmShift shift = lsl; 2680 switch (code) { 2681 case lir_shl: shift = lsl; break; 2682 case lir_shr: shift = asr; break; 2683 case lir_ushr: shift = lsr; break; 2684 default: ShouldNotReachHere(); 2685 } 2686 2687 if (dest->is_single_cpu()) { 2688 count &= 31; 2689 if (count != 0) { 2690 __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, count)); 2691 } else { 2692 move_regs(left->as_register(), dest->as_register()); 2693 } 2694 } else if (dest->is_double_cpu()) { 2695 count &= 63; 2696 if (count != 0) { 2697 Register dest_lo = dest->as_register_lo(); 2698 Register dest_hi = dest->as_register_hi(); 2699 Register src_lo = left->as_register_lo(); 2700 Register src_hi = left->as_register_hi(); 2701 // Resolve possible register conflicts 2702 if (shift == lsl && dest_hi == src_lo) { 2703 dest_hi = Rtemp; 2704 } else if (shift != lsl && dest_lo == src_hi) { 2705 dest_lo = Rtemp; 2706 } 2707 __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, count); 2708 move_regs(dest_lo, dest->as_register_lo()); 2709 move_regs(dest_hi, dest->as_register_hi()); 2710 } else { 2711 __ long_move(dest->as_register_lo(), dest->as_register_hi(), 2712 left->as_register_lo(), left->as_register_hi()); 2713 } 2714 } else { 2715 ShouldNotReachHere(); 2716 } 2717 #endif // AARCH64 2718 } 2719 2720 2721 // Saves 4 given registers in reserved argument area. 2722 void LIR_Assembler::save_in_reserved_area(Register r1, Register r2, Register r3, Register r4) { 2723 verify_reserved_argument_area_size(4); 2724 #ifdef AARCH64 2725 __ stp(r1, r2, Address(SP, 0)); 2726 __ stp(r3, r4, Address(SP, 2*wordSize)); 2727 #else 2728 __ stmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4)); 2729 #endif // AARCH64 2730 } 2731 2732 // Restores 4 given registers from reserved argument area. 2733 void LIR_Assembler::restore_from_reserved_area(Register r1, Register r2, Register r3, Register r4) { 2734 #ifdef AARCH64 2735 __ ldp(r1, r2, Address(SP, 0)); 2736 __ ldp(r3, r4, Address(SP, 2*wordSize)); 2737 #else 2738 __ ldmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4), no_writeback); 2739 #endif // AARCH64 2740 } 2741 2742 2743 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 2744 ciArrayKlass* default_type = op->expected_type(); 2745 Register src = op->src()->as_register(); 2746 Register src_pos = op->src_pos()->as_register(); 2747 Register dst = op->dst()->as_register(); 2748 Register dst_pos = op->dst_pos()->as_register(); 2749 Register length = op->length()->as_register(); 2750 Register tmp = op->tmp()->as_register(); 2751 Register tmp2 = Rtemp; 2752 2753 assert(src == R0 && src_pos == R1 && dst == R2 && dst_pos == R3, "code assumption"); 2754 #ifdef AARCH64 2755 assert(length == R4, "code assumption"); 2756 #endif // AARCH64 2757 2758 CodeStub* stub = op->stub(); 2759 2760 int flags = op->flags(); 2761 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL; 2762 if (basic_type == T_ARRAY) basic_type = T_OBJECT; 2763 2764 // If we don't know anything or it's an object array, just go through the generic arraycopy 2765 if (default_type == NULL) { 2766 2767 // save arguments, because they will be killed by a runtime call 2768 save_in_reserved_area(R0, R1, R2, R3); 2769 2770 #ifdef AARCH64 2771 // save length argument, will be killed by a runtime call 2772 __ raw_push(length, ZR); 2773 #else 2774 // pass length argument on SP[0] 2775 __ str(length, Address(SP, -2*wordSize, pre_indexed)); // 2 words for a proper stack alignment 2776 #endif // AARCH64 2777 2778 address copyfunc_addr = StubRoutines::generic_arraycopy(); 2779 if (copyfunc_addr == NULL) { // Use C version if stub was not generated 2780 __ call(CAST_FROM_FN_PTR(address, Runtime1::arraycopy)); 2781 } else { 2782 #ifndef PRODUCT 2783 if (PrintC1Statistics) { 2784 __ inc_counter((address)&Runtime1::_generic_arraycopystub_cnt, tmp, tmp2); 2785 } 2786 #endif // !PRODUCT 2787 // the stub is in the code cache so close enough 2788 __ call(copyfunc_addr, relocInfo::runtime_call_type); 2789 } 2790 2791 #ifdef AARCH64 2792 __ raw_pop(length, ZR); 2793 #else 2794 __ add(SP, SP, 2*wordSize); 2795 #endif // AARCH64 2796 2797 __ cbz_32(R0, *stub->continuation()); 2798 2799 if (copyfunc_addr != NULL) { 2800 __ mvn_32(tmp, R0); 2801 restore_from_reserved_area(R0, R1, R2, R3); // load saved arguments in slow case only 2802 __ sub_32(length, length, tmp); 2803 __ add_32(src_pos, src_pos, tmp); 2804 __ add_32(dst_pos, dst_pos, tmp); 2805 } else { 2806 restore_from_reserved_area(R0, R1, R2, R3); // load saved arguments in slow case only 2807 } 2808 2809 __ b(*stub->entry()); 2810 2811 __ bind(*stub->continuation()); 2812 return; 2813 } 2814 2815 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), 2816 "must be true at this point"); 2817 int elem_size = type2aelembytes(basic_type); 2818 int shift = exact_log2(elem_size); 2819 2820 // Check for NULL 2821 if (flags & LIR_OpArrayCopy::src_null_check) { 2822 if (flags & LIR_OpArrayCopy::dst_null_check) { 2823 __ cmp(src, 0); 2824 __ cond_cmp(dst, 0, ne); // make one instruction shorter if both checks are needed 2825 __ b(*stub->entry(), eq); 2826 } else { 2827 __ cbz(src, *stub->entry()); 2828 } 2829 } else if (flags & LIR_OpArrayCopy::dst_null_check) { 2830 __ cbz(dst, *stub->entry()); 2831 } 2832 2833 // If the compiler was not able to prove that exact type of the source or the destination 2834 // of the arraycopy is an array type, check at runtime if the source or the destination is 2835 // an instance type. 2836 if (flags & LIR_OpArrayCopy::type_check) { 2837 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) { 2838 __ load_klass(tmp, dst); 2839 __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2840 __ mov_slow(tmp, Klass::_lh_neutral_value); 2841 __ cmp_32(tmp2, tmp); 2842 __ b(*stub->entry(), ge); 2843 } 2844 2845 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) { 2846 __ load_klass(tmp, src); 2847 __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2848 __ mov_slow(tmp, Klass::_lh_neutral_value); 2849 __ cmp_32(tmp2, tmp); 2850 __ b(*stub->entry(), ge); 2851 } 2852 } 2853 2854 // Check if negative 2855 const int all_positive_checks = LIR_OpArrayCopy::src_pos_positive_check | 2856 LIR_OpArrayCopy::dst_pos_positive_check | 2857 LIR_OpArrayCopy::length_positive_check; 2858 switch (flags & all_positive_checks) { 2859 case LIR_OpArrayCopy::src_pos_positive_check: 2860 __ branch_if_negative_32(src_pos, *stub->entry()); 2861 break; 2862 case LIR_OpArrayCopy::dst_pos_positive_check: 2863 __ branch_if_negative_32(dst_pos, *stub->entry()); 2864 break; 2865 case LIR_OpArrayCopy::length_positive_check: 2866 __ branch_if_negative_32(length, *stub->entry()); 2867 break; 2868 case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::dst_pos_positive_check: 2869 __ branch_if_any_negative_32(src_pos, dst_pos, tmp, *stub->entry()); 2870 break; 2871 case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::length_positive_check: 2872 __ branch_if_any_negative_32(src_pos, length, tmp, *stub->entry()); 2873 break; 2874 case LIR_OpArrayCopy::dst_pos_positive_check | LIR_OpArrayCopy::length_positive_check: 2875 __ branch_if_any_negative_32(dst_pos, length, tmp, *stub->entry()); 2876 break; 2877 case all_positive_checks: 2878 __ branch_if_any_negative_32(src_pos, dst_pos, length, tmp, *stub->entry()); 2879 break; 2880 default: 2881 assert((flags & all_positive_checks) == 0, "the last option"); 2882 } 2883 2884 // Range checks 2885 if (flags & LIR_OpArrayCopy::src_range_check) { 2886 __ ldr_s32(tmp2, Address(src, arrayOopDesc::length_offset_in_bytes())); 2887 __ add_32(tmp, src_pos, length); 2888 __ cmp_32(tmp, tmp2); 2889 __ b(*stub->entry(), hi); 2890 } 2891 if (flags & LIR_OpArrayCopy::dst_range_check) { 2892 __ ldr_s32(tmp2, Address(dst, arrayOopDesc::length_offset_in_bytes())); 2893 __ add_32(tmp, dst_pos, length); 2894 __ cmp_32(tmp, tmp2); 2895 __ b(*stub->entry(), hi); 2896 } 2897 2898 // Check if src and dst are of the same type 2899 if (flags & LIR_OpArrayCopy::type_check) { 2900 // We don't know the array types are compatible 2901 if (basic_type != T_OBJECT) { 2902 // Simple test for basic type arrays 2903 if (UseCompressedClassPointers) { 2904 // We don't need decode because we just need to compare 2905 __ ldr_u32(tmp, Address(src, oopDesc::klass_offset_in_bytes())); 2906 __ ldr_u32(tmp2, Address(dst, oopDesc::klass_offset_in_bytes())); 2907 __ cmp_32(tmp, tmp2); 2908 } else { 2909 __ load_klass(tmp, src); 2910 __ load_klass(tmp2, dst); 2911 __ cmp(tmp, tmp2); 2912 } 2913 __ b(*stub->entry(), ne); 2914 } else { 2915 // For object arrays, if src is a sub class of dst then we can 2916 // safely do the copy. 2917 Label cont, slow; 2918 2919 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 2920 2921 __ load_klass(tmp, src); 2922 __ load_klass(tmp2, dst); 2923 2924 // We are at a call so all live registers are saved before we 2925 // get here 2926 assert_different_registers(tmp, tmp2, R6, altFP_7_11); 2927 2928 __ check_klass_subtype_fast_path(tmp, tmp2, R6, altFP_7_11, &cont, copyfunc_addr == NULL ? stub->entry() : &slow, NULL); 2929 2930 __ mov(R6, R0); 2931 __ mov(altFP_7_11, R1); 2932 __ mov(R0, tmp); 2933 __ mov(R1, tmp2); 2934 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); // does not blow any registers except R0, LR and Rtemp 2935 __ cmp_32(R0, 0); 2936 __ mov(R0, R6); 2937 __ mov(R1, altFP_7_11); 2938 2939 if (copyfunc_addr != NULL) { // use stub if available 2940 // src is not a sub class of dst so we have to do a 2941 // per-element check. 2942 2943 __ b(cont, ne); 2944 2945 __ bind(slow); 2946 2947 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 2948 if ((flags & mask) != mask) { 2949 // Check that at least both of them object arrays. 2950 assert(flags & mask, "one of the two should be known to be an object array"); 2951 2952 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 2953 __ load_klass(tmp, src); 2954 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 2955 __ load_klass(tmp, dst); 2956 } 2957 int lh_offset = in_bytes(Klass::layout_helper_offset()); 2958 2959 __ ldr_u32(tmp2, Address(tmp, lh_offset)); 2960 2961 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 2962 __ mov_slow(tmp, objArray_lh); 2963 __ cmp_32(tmp, tmp2); 2964 __ b(*stub->entry(), ne); 2965 } 2966 2967 save_in_reserved_area(R0, R1, R2, R3); 2968 2969 Register src_ptr = R0; 2970 Register dst_ptr = R1; 2971 Register len = R2; 2972 Register chk_off = R3; 2973 Register super_k = AARCH64_ONLY(R4) NOT_AARCH64(tmp); 2974 2975 __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type)); 2976 __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift); 2977 2978 __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type)); 2979 __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift); 2980 __ load_klass(tmp, dst); 2981 2982 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); 2983 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 2984 2985 #ifdef AARCH64 2986 __ raw_push(length, ZR); // Preserve length around *copyfunc_addr call 2987 2988 __ mov(len, length); 2989 __ ldr(super_k, Address(tmp, ek_offset)); // super_k == R4 == length, so this load cannot be performed earlier 2990 // TODO-AARCH64: check whether it is faster to load super klass early by using tmp and additional mov. 2991 __ ldr_u32(chk_off, Address(super_k, sco_offset)); 2992 #else // AARCH64 2993 __ ldr(super_k, Address(tmp, ek_offset)); 2994 2995 __ mov(len, length); 2996 __ ldr_u32(chk_off, Address(super_k, sco_offset)); 2997 __ push(super_k); 2998 #endif // AARCH64 2999 3000 __ call(copyfunc_addr, relocInfo::runtime_call_type); 3001 3002 #ifndef PRODUCT 3003 if (PrintC1Statistics) { 3004 Label failed; 3005 __ cbnz_32(R0, failed); 3006 __ inc_counter((address)&Runtime1::_arraycopy_checkcast_cnt, tmp, tmp2); 3007 __ bind(failed); 3008 } 3009 #endif // PRODUCT 3010 3011 #ifdef AARCH64 3012 __ raw_pop(length, ZR); 3013 #else 3014 __ add(SP, SP, wordSize); // Drop super_k argument 3015 #endif // AARCH64 3016 3017 __ cbz_32(R0, *stub->continuation()); 3018 __ mvn_32(tmp, R0); 3019 3020 // load saved arguments in slow case only 3021 restore_from_reserved_area(R0, R1, R2, R3); 3022 3023 __ sub_32(length, length, tmp); 3024 __ add_32(src_pos, src_pos, tmp); 3025 __ add_32(dst_pos, dst_pos, tmp); 3026 3027 #ifndef PRODUCT 3028 if (PrintC1Statistics) { 3029 __ inc_counter((address)&Runtime1::_arraycopy_checkcast_attempt_cnt, tmp, tmp2); 3030 } 3031 #endif 3032 3033 __ b(*stub->entry()); 3034 3035 __ bind(cont); 3036 } else { 3037 __ b(*stub->entry(), eq); 3038 __ bind(cont); 3039 } 3040 } 3041 } 3042 3043 #ifndef PRODUCT 3044 if (PrintC1Statistics) { 3045 address counter = Runtime1::arraycopy_count_address(basic_type); 3046 __ inc_counter(counter, tmp, tmp2); 3047 } 3048 #endif // !PRODUCT 3049 3050 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 3051 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 3052 const char *name; 3053 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 3054 3055 Register src_ptr = R0; 3056 Register dst_ptr = R1; 3057 Register len = R2; 3058 3059 __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type)); 3060 __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift); 3061 3062 __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type)); 3063 __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift); 3064 3065 __ mov(len, length); 3066 3067 __ call(entry, relocInfo::runtime_call_type); 3068 3069 __ bind(*stub->continuation()); 3070 } 3071 3072 #ifdef ASSERT 3073 // emit run-time assertion 3074 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 3075 assert(op->code() == lir_assert, "must be"); 3076 3077 #ifdef AARCH64 3078 __ NOT_IMPLEMENTED(); 3079 #else 3080 if (op->in_opr1()->is_valid()) { 3081 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 3082 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 3083 } else { 3084 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 3085 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 3086 } 3087 3088 Label ok; 3089 if (op->condition() != lir_cond_always) { 3090 AsmCondition acond; 3091 switch (op->condition()) { 3092 case lir_cond_equal: acond = eq; break; 3093 case lir_cond_notEqual: acond = ne; break; 3094 case lir_cond_less: acond = lt; break; 3095 case lir_cond_lessEqual: acond = le; break; 3096 case lir_cond_greaterEqual: acond = ge; break; 3097 case lir_cond_greater: acond = gt; break; 3098 case lir_cond_aboveEqual: acond = hs; break; 3099 case lir_cond_belowEqual: acond = ls; break; 3100 default: ShouldNotReachHere(); 3101 } 3102 __ b(ok, acond); 3103 } 3104 if (op->halt()) { 3105 const char* str = __ code_string(op->msg()); 3106 __ stop(str); 3107 } else { 3108 breakpoint(); 3109 } 3110 __ bind(ok); 3111 #endif // AARCH64 3112 } 3113 #endif // ASSERT 3114 3115 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 3116 fatal("CRC32 intrinsic is not implemented on this platform"); 3117 } 3118 3119 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 3120 Register obj = op->obj_opr()->as_pointer_register(); 3121 Register hdr = op->hdr_opr()->as_pointer_register(); 3122 Register lock = op->lock_opr()->as_pointer_register(); 3123 Register tmp = op->scratch_opr()->is_illegal() ? noreg : 3124 op->scratch_opr()->as_pointer_register(); 3125 3126 if (!UseFastLocking) { 3127 __ b(*op->stub()->entry()); 3128 } else if (op->code() == lir_lock) { 3129 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 3130 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry()); 3131 if (op->info() != NULL) { 3132 add_debug_info_for_null_check(null_check_offset, op->info()); 3133 } 3134 } else if (op->code() == lir_unlock) { 3135 __ unlock_object(hdr, obj, lock, tmp, *op->stub()->entry()); 3136 } else { 3137 ShouldNotReachHere(); 3138 } 3139 __ bind(*op->stub()->continuation()); 3140 } 3141 3142 3143 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 3144 ciMethod* method = op->profiled_method(); 3145 int bci = op->profiled_bci(); 3146 ciMethod* callee = op->profiled_callee(); 3147 3148 // Update counter for all call types 3149 ciMethodData* md = method->method_data_or_null(); 3150 assert(md != NULL, "Sanity"); 3151 ciProfileData* data = md->bci_to_data(bci); 3152 assert(data->is_CounterData(), "need CounterData for calls"); 3153 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 3154 Register mdo = op->mdo()->as_register(); 3155 assert(op->tmp1()->is_register(), "tmp1 must be allocated"); 3156 Register tmp1 = op->tmp1()->as_pointer_register(); 3157 assert_different_registers(mdo, tmp1); 3158 __ mov_metadata(mdo, md->constant_encoding()); 3159 int mdo_offset_bias = 0; 3160 int max_offset = AARCH64_ONLY(4096 << LogBytesPerWord) NOT_AARCH64(4096); 3161 if (md->byte_offset_of_slot(data, CounterData::count_offset()) + data->size_in_bytes() >= max_offset) { 3162 // The offset is large so bias the mdo by the base of the slot so 3163 // that the ldr can use an immediate offset to reference the slots of the data 3164 mdo_offset_bias = md->byte_offset_of_slot(data, CounterData::count_offset()); 3165 __ mov_slow(tmp1, mdo_offset_bias); 3166 __ add(mdo, mdo, tmp1); 3167 } 3168 3169 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias); 3170 Bytecodes::Code bc = method->java_code_at_bci(bci); 3171 const bool callee_is_static = callee->is_loaded() && callee->is_static(); 3172 // Perform additional virtual call profiling for invokevirtual and 3173 // invokeinterface bytecodes 3174 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) && 3175 !callee_is_static && // required for optimized MH invokes 3176 C1ProfileVirtualCalls) { 3177 3178 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 3179 Register recv = op->recv()->as_register(); 3180 assert_different_registers(mdo, tmp1, recv); 3181 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 3182 ciKlass* known_klass = op->known_holder(); 3183 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) { 3184 // We know the type that will be seen at this call site; we can 3185 // statically update the MethodData* rather than needing to do 3186 // dynamic tests on the receiver type 3187 3188 // NOTE: we should probably put a lock around this search to 3189 // avoid collisions by concurrent compilations 3190 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 3191 uint i; 3192 for (i = 0; i < VirtualCallData::row_limit(); i++) { 3193 ciKlass* receiver = vc_data->receiver(i); 3194 if (known_klass->equals(receiver)) { 3195 Address data_addr(mdo, md->byte_offset_of_slot(data, 3196 VirtualCallData::receiver_count_offset(i)) - 3197 mdo_offset_bias); 3198 __ ldr(tmp1, data_addr); 3199 __ add(tmp1, tmp1, DataLayout::counter_increment); 3200 __ str(tmp1, data_addr); 3201 return; 3202 } 3203 } 3204 3205 // Receiver type not found in profile data; select an empty slot 3206 3207 // Note that this is less efficient than it should be because it 3208 // always does a write to the receiver part of the 3209 // VirtualCallData rather than just the first time 3210 for (i = 0; i < VirtualCallData::row_limit(); i++) { 3211 ciKlass* receiver = vc_data->receiver(i); 3212 if (receiver == NULL) { 3213 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) - 3214 mdo_offset_bias); 3215 __ mov_metadata(tmp1, known_klass->constant_encoding()); 3216 __ str(tmp1, recv_addr); 3217 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) - 3218 mdo_offset_bias); 3219 __ ldr(tmp1, data_addr); 3220 __ add(tmp1, tmp1, DataLayout::counter_increment); 3221 __ str(tmp1, data_addr); 3222 return; 3223 } 3224 } 3225 } else { 3226 __ load_klass(recv, recv); 3227 Label update_done; 3228 type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &update_done); 3229 // Receiver did not match any saved receiver and there is no empty row for it. 3230 // Increment total counter to indicate polymorphic case. 3231 __ ldr(tmp1, counter_addr); 3232 __ add(tmp1, tmp1, DataLayout::counter_increment); 3233 __ str(tmp1, counter_addr); 3234 3235 __ bind(update_done); 3236 } 3237 } else { 3238 // Static call 3239 __ ldr(tmp1, counter_addr); 3240 __ add(tmp1, tmp1, DataLayout::counter_increment); 3241 __ str(tmp1, counter_addr); 3242 } 3243 } 3244 3245 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 3246 fatal("Type profiling not implemented on this platform"); 3247 } 3248 3249 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 3250 Unimplemented(); 3251 } 3252 3253 3254 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 3255 Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no); 3256 __ add_slow(dst->as_pointer_register(), mon_addr.base(), mon_addr.disp()); 3257 } 3258 3259 3260 void LIR_Assembler::align_backward_branch_target() { 3261 // TODO-AARCH64 review it 3262 // Some ARM processors do better with 8-byte branch target alignment 3263 __ align(8); 3264 } 3265 3266 3267 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) { 3268 3269 if (left->is_single_cpu()) { 3270 assert (dest->type() == T_INT, "unexpected result type"); 3271 assert (left->type() == T_INT, "unexpected left type"); 3272 __ neg_32(dest->as_register(), left->as_register()); 3273 } else if (left->is_double_cpu()) { 3274 #ifdef AARCH64 3275 __ neg(dest->as_register_lo(), left->as_register_lo()); 3276 #else 3277 Register dest_lo = dest->as_register_lo(); 3278 Register dest_hi = dest->as_register_hi(); 3279 Register src_lo = left->as_register_lo(); 3280 Register src_hi = left->as_register_hi(); 3281 if (dest_lo == src_hi) { 3282 dest_lo = Rtemp; 3283 } 3284 __ rsbs(dest_lo, src_lo, 0); 3285 __ rsc(dest_hi, src_hi, 0); 3286 move_regs(dest_lo, dest->as_register_lo()); 3287 #endif // AARCH64 3288 } else if (left->is_single_fpu()) { 3289 __ neg_float(dest->as_float_reg(), left->as_float_reg()); 3290 } else if (left->is_double_fpu()) { 3291 __ neg_double(dest->as_double_reg(), left->as_double_reg()); 3292 } else { 3293 ShouldNotReachHere(); 3294 } 3295 } 3296 3297 3298 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest) { 3299 LIR_Address* addr = addr_opr->as_address_ptr(); 3300 if (addr->index()->is_illegal()) { 3301 jint c = addr->disp(); 3302 if (!Assembler::is_arith_imm_in_range(c)) { 3303 BAILOUT("illegal arithmetic operand"); 3304 } 3305 __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(), c); 3306 } else { 3307 assert(addr->disp() == 0, "cannot handle otherwise"); 3308 #ifdef AARCH64 3309 assert(addr->index()->is_double_cpu(), "should be"); 3310 #endif // AARCH64 3311 __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(), 3312 AsmOperand(addr->index()->as_pointer_register(), lsl, addr->scale())); 3313 } 3314 } 3315 3316 3317 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 3318 assert(!tmp->is_valid(), "don't need temporary"); 3319 __ call(dest); 3320 if (info != NULL) { 3321 add_call_info_here(info); 3322 } 3323 } 3324 3325 3326 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 3327 #ifdef AARCH64 3328 Unimplemented(); // TODO-AARCH64: Use stlr/ldar instructions for volatile load/store 3329 #else 3330 assert(src->is_double_cpu() && dest->is_address() || 3331 src->is_address() && dest->is_double_cpu(), 3332 "Simple move_op is called for all other cases"); 3333 3334 int null_check_offset; 3335 if (dest->is_address()) { 3336 // Store 3337 const LIR_Address* addr = dest->as_address_ptr(); 3338 const Register src_lo = src->as_register_lo(); 3339 const Register src_hi = src->as_register_hi(); 3340 assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already"); 3341 3342 if (src_lo < src_hi) { 3343 null_check_offset = __ offset(); 3344 __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(src_hi)); 3345 } else { 3346 assert(src_lo < Rtemp, "Rtemp is higher than any allocatable register"); 3347 __ mov(Rtemp, src_hi); 3348 null_check_offset = __ offset(); 3349 __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(Rtemp)); 3350 } 3351 } else { 3352 // Load 3353 const LIR_Address* addr = src->as_address_ptr(); 3354 const Register dest_lo = dest->as_register_lo(); 3355 const Register dest_hi = dest->as_register_hi(); 3356 assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already"); 3357 3358 null_check_offset = __ offset(); 3359 if (dest_lo < dest_hi) { 3360 __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(dest_hi)); 3361 } else { 3362 assert(dest_lo < Rtemp, "Rtemp is higher than any allocatable register"); 3363 __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(Rtemp)); 3364 __ mov(dest_hi, Rtemp); 3365 } 3366 } 3367 3368 if (info != NULL) { 3369 add_debug_info_for_null_check(null_check_offset, info); 3370 } 3371 #endif // AARCH64 3372 } 3373 3374 3375 void LIR_Assembler::membar() { 3376 __ membar(MacroAssembler::StoreLoad, Rtemp); 3377 } 3378 3379 void LIR_Assembler::membar_acquire() { 3380 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp); 3381 } 3382 3383 void LIR_Assembler::membar_release() { 3384 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp); 3385 } 3386 3387 void LIR_Assembler::membar_loadload() { 3388 __ membar(MacroAssembler::LoadLoad, Rtemp); 3389 } 3390 3391 void LIR_Assembler::membar_storestore() { 3392 __ membar(MacroAssembler::StoreStore, Rtemp); 3393 } 3394 3395 void LIR_Assembler::membar_loadstore() { 3396 __ membar(MacroAssembler::LoadStore, Rtemp); 3397 } 3398 3399 void LIR_Assembler::membar_storeload() { 3400 __ membar(MacroAssembler::StoreLoad, Rtemp); 3401 } 3402 3403 void LIR_Assembler::on_spin_wait() { 3404 Unimplemented(); 3405 } 3406 3407 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 3408 // Not used on ARM 3409 Unimplemented(); 3410 } 3411 3412 void LIR_Assembler::peephole(LIR_List* lir) { 3413 #ifdef AARCH64 3414 return; // TODO-AARCH64 implement peephole optimizations 3415 #endif 3416 LIR_OpList* inst = lir->instructions_list(); 3417 const int inst_length = inst->length(); 3418 for (int i = 0; i < inst_length; i++) { 3419 LIR_Op* op = inst->at(i); 3420 switch (op->code()) { 3421 case lir_cmp: { 3422 // Replace: 3423 // cmp rX, y 3424 // cmove [EQ] y, z, rX 3425 // with 3426 // cmp rX, y 3427 // cmove [EQ] illegalOpr, z, rX 3428 // 3429 // or 3430 // cmp rX, y 3431 // cmove [NE] z, y, rX 3432 // with 3433 // cmp rX, y 3434 // cmove [NE] z, illegalOpr, rX 3435 // 3436 // moves from illegalOpr should be removed when converting LIR to native assembly 3437 3438 LIR_Op2* cmp = op->as_Op2(); 3439 assert(cmp != NULL, "cmp LIR instruction is not an op2"); 3440 3441 if (i + 1 < inst_length) { 3442 LIR_Op2* cmove = inst->at(i + 1)->as_Op2(); 3443 if (cmove != NULL && cmove->code() == lir_cmove) { 3444 LIR_Opr cmove_res = cmove->result_opr(); 3445 bool res_is_op1 = cmove_res == cmp->in_opr1(); 3446 bool res_is_op2 = cmove_res == cmp->in_opr2(); 3447 LIR_Opr cmp_res, cmp_arg; 3448 if (res_is_op1) { 3449 cmp_res = cmp->in_opr1(); 3450 cmp_arg = cmp->in_opr2(); 3451 } else if (res_is_op2) { 3452 cmp_res = cmp->in_opr2(); 3453 cmp_arg = cmp->in_opr1(); 3454 } else { 3455 cmp_res = LIR_OprFact::illegalOpr; 3456 cmp_arg = LIR_OprFact::illegalOpr; 3457 } 3458 3459 if (cmp_res != LIR_OprFact::illegalOpr) { 3460 LIR_Condition cond = cmove->condition(); 3461 if (cond == lir_cond_equal && cmove->in_opr1() == cmp_arg) { 3462 cmove->set_in_opr1(LIR_OprFact::illegalOpr); 3463 } else if (cond == lir_cond_notEqual && cmove->in_opr2() == cmp_arg) { 3464 cmove->set_in_opr2(LIR_OprFact::illegalOpr); 3465 } 3466 } 3467 } 3468 } 3469 break; 3470 } 3471 3472 default: 3473 break; 3474 } 3475 } 3476 } 3477 3478 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) { 3479 Register ptr = src->as_pointer_register(); 3480 3481 if (code == lir_xchg) { 3482 #ifdef AARCH64 3483 if (UseCompressedOops && data->is_oop()) { 3484 __ encode_heap_oop(tmp->as_pointer_register(), data->as_register()); 3485 } 3486 #endif // AARCH64 3487 } else { 3488 assert (!data->is_oop(), "xadd for oops"); 3489 } 3490 3491 #ifndef AARCH64 3492 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp); 3493 #endif // !AARCH64 3494 3495 Label retry; 3496 __ bind(retry); 3497 3498 if ((data->type() == T_INT) || (data->is_oop() AARCH64_ONLY(&& UseCompressedOops))) { 3499 Register dst = dest->as_register(); 3500 Register new_val = noreg; 3501 #ifdef AARCH64 3502 __ ldaxr_w(dst, ptr); 3503 #else 3504 __ ldrex(dst, Address(ptr)); 3505 #endif 3506 if (code == lir_xadd) { 3507 Register tmp_reg = tmp->as_register(); 3508 if (data->is_constant()) { 3509 assert_different_registers(dst, ptr, tmp_reg); 3510 __ add_32(tmp_reg, dst, data->as_constant_ptr()->as_jint()); 3511 } else { 3512 assert_different_registers(dst, ptr, tmp_reg, data->as_register()); 3513 __ add_32(tmp_reg, dst, data->as_register()); 3514 } 3515 new_val = tmp_reg; 3516 } else { 3517 if (UseCompressedOops && data->is_oop()) { 3518 new_val = tmp->as_pointer_register(); 3519 } else { 3520 new_val = data->as_register(); 3521 } 3522 assert_different_registers(dst, ptr, new_val); 3523 } 3524 #ifdef AARCH64 3525 __ stlxr_w(Rtemp, new_val, ptr); 3526 #else 3527 __ strex(Rtemp, new_val, Address(ptr)); 3528 #endif // AARCH64 3529 3530 #ifdef AARCH64 3531 } else if ((data->type() == T_LONG) || (data->is_oop() && !UseCompressedOops)) { 3532 Register dst = dest->as_pointer_register(); 3533 Register new_val = noreg; 3534 __ ldaxr(dst, ptr); 3535 if (code == lir_xadd) { 3536 Register tmp_reg = tmp->as_pointer_register(); 3537 if (data->is_constant()) { 3538 assert_different_registers(dst, ptr, tmp_reg); 3539 jlong c = data->as_constant_ptr()->as_jlong(); 3540 assert((jlong)((jint)c) == c, "overflow"); 3541 __ add(tmp_reg, dst, (jint)c); 3542 } else { 3543 assert_different_registers(dst, ptr, tmp_reg, data->as_pointer_register()); 3544 __ add(tmp_reg, dst, data->as_pointer_register()); 3545 } 3546 new_val = tmp_reg; 3547 } else { 3548 new_val = data->as_pointer_register(); 3549 assert_different_registers(dst, ptr, new_val); 3550 } 3551 __ stlxr(Rtemp, new_val, ptr); 3552 #else 3553 } else if (data->type() == T_LONG) { 3554 Register dst_lo = dest->as_register_lo(); 3555 Register new_val_lo = noreg; 3556 Register dst_hi = dest->as_register_hi(); 3557 3558 assert(dst_hi->encoding() == dst_lo->encoding() + 1, "non aligned register pair"); 3559 assert((dst_lo->encoding() & 0x1) == 0, "misaligned register pair"); 3560 3561 __ bind(retry); 3562 __ ldrexd(dst_lo, Address(ptr)); 3563 if (code == lir_xadd) { 3564 Register tmp_lo = tmp->as_register_lo(); 3565 Register tmp_hi = tmp->as_register_hi(); 3566 3567 assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair"); 3568 assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair"); 3569 3570 if (data->is_constant()) { 3571 jlong c = data->as_constant_ptr()->as_jlong(); 3572 assert((jlong)((jint)c) == c, "overflow"); 3573 assert_different_registers(dst_lo, dst_hi, ptr, tmp_lo, tmp_hi); 3574 __ adds(tmp_lo, dst_lo, (jint)c); 3575 __ adc(tmp_hi, dst_hi, 0); 3576 } else { 3577 Register new_val_lo = data->as_register_lo(); 3578 Register new_val_hi = data->as_register_hi(); 3579 __ adds(tmp_lo, dst_lo, new_val_lo); 3580 __ adc(tmp_hi, dst_hi, new_val_hi); 3581 assert_different_registers(dst_lo, dst_hi, ptr, tmp_lo, tmp_hi, new_val_lo, new_val_hi); 3582 } 3583 new_val_lo = tmp_lo; 3584 } else { 3585 new_val_lo = data->as_register_lo(); 3586 Register new_val_hi = data->as_register_hi(); 3587 3588 assert_different_registers(dst_lo, dst_hi, ptr, new_val_lo, new_val_hi); 3589 assert(new_val_hi->encoding() == new_val_lo->encoding() + 1, "non aligned register pair"); 3590 assert((new_val_lo->encoding() & 0x1) == 0, "misaligned register pair"); 3591 } 3592 __ strexd(Rtemp, new_val_lo, Address(ptr)); 3593 #endif // AARCH64 3594 } else { 3595 ShouldNotReachHere(); 3596 } 3597 3598 __ cbnz_32(Rtemp, retry); 3599 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp); 3600 3601 #ifdef AARCH64 3602 if (UseCompressedOops && data->is_oop()) { 3603 __ decode_heap_oop(dest->as_register()); 3604 } 3605 #endif // AARCH64 3606 } 3607 3608 #undef __