1 /* 2 * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, Red Hat Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "asm/assembler.hpp" 28 #include "c1/c1_CodeStubs.hpp" 29 #include "c1/c1_Compilation.hpp" 30 #include "c1/c1_LIRAssembler.hpp" 31 #include "c1/c1_MacroAssembler.hpp" 32 #include "c1/c1_Runtime1.hpp" 33 #include "c1/c1_ValueStack.hpp" 34 #include "ci/ciArrayKlass.hpp" 35 #include "ci/ciInstance.hpp" 36 #include "gc/shared/barrierSet.hpp" 37 #include "gc/shared/cardTableModRefBS.hpp" 38 #include "gc/shared/collectedHeap.hpp" 39 #include "nativeInst_aarch64.hpp" 40 #include "oops/objArrayKlass.hpp" 41 #include "runtime/sharedRuntime.hpp" 42 #include "vmreg_aarch64.inline.hpp" 43 44 45 46 #ifndef PRODUCT 47 #define COMMENT(x) do { __ block_comment(x); } while (0) 48 #else 49 #define COMMENT(x) 50 #endif 51 52 NEEDS_CLEANUP // remove this definitions ? 53 const Register IC_Klass = rscratch2; // where the IC klass is cached 54 const Register SYNC_header = r0; // synchronization header 55 const Register SHIFT_count = r0; // where count for shift operations must be 56 57 #define __ _masm-> 58 59 60 static void select_different_registers(Register preserve, 61 Register extra, 62 Register &tmp1, 63 Register &tmp2) { 64 if (tmp1 == preserve) { 65 assert_different_registers(tmp1, tmp2, extra); 66 tmp1 = extra; 67 } else if (tmp2 == preserve) { 68 assert_different_registers(tmp1, tmp2, extra); 69 tmp2 = extra; 70 } 71 assert_different_registers(preserve, tmp1, tmp2); 72 } 73 74 75 76 static void select_different_registers(Register preserve, 77 Register extra, 78 Register &tmp1, 79 Register &tmp2, 80 Register &tmp3) { 81 if (tmp1 == preserve) { 82 assert_different_registers(tmp1, tmp2, tmp3, extra); 83 tmp1 = extra; 84 } else if (tmp2 == preserve) { 85 assert_different_registers(tmp1, tmp2, tmp3, extra); 86 tmp2 = extra; 87 } else if (tmp3 == preserve) { 88 assert_different_registers(tmp1, tmp2, tmp3, extra); 89 tmp3 = extra; 90 } 91 assert_different_registers(preserve, tmp1, tmp2, tmp3); 92 } 93 94 95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } 96 97 98 LIR_Opr LIR_Assembler::receiverOpr() { 99 return FrameMap::receiver_opr; 100 } 101 102 LIR_Opr LIR_Assembler::osrBufferPointer() { 103 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 104 } 105 106 //--------------fpu register translations----------------------- 107 108 109 address LIR_Assembler::float_constant(float f) { 110 address const_addr = __ float_constant(f); 111 if (const_addr == NULL) { 112 bailout("const section overflow"); 113 return __ code()->consts()->start(); 114 } else { 115 return const_addr; 116 } 117 } 118 119 120 address LIR_Assembler::double_constant(double d) { 121 address const_addr = __ double_constant(d); 122 if (const_addr == NULL) { 123 bailout("const section overflow"); 124 return __ code()->consts()->start(); 125 } else { 126 return const_addr; 127 } 128 } 129 130 address LIR_Assembler::int_constant(jlong n) { 131 address const_addr = __ long_constant(n); 132 if (const_addr == NULL) { 133 bailout("const section overflow"); 134 return __ code()->consts()->start(); 135 } else { 136 return const_addr; 137 } 138 } 139 140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); } 141 142 void LIR_Assembler::reset_FPU() { Unimplemented(); } 143 144 void LIR_Assembler::fpop() { Unimplemented(); } 145 146 void LIR_Assembler::fxch(int i) { Unimplemented(); } 147 148 void LIR_Assembler::fld(int i) { Unimplemented(); } 149 150 void LIR_Assembler::ffree(int i) { Unimplemented(); } 151 152 void LIR_Assembler::breakpoint() { Unimplemented(); } 153 154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); } 155 156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); } 157 158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; } 159 //------------------------------------------- 160 161 static Register as_reg(LIR_Opr op) { 162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register(); 163 } 164 165 static jlong as_long(LIR_Opr data) { 166 jlong result; 167 switch (data->type()) { 168 case T_INT: 169 result = (data->as_jint()); 170 break; 171 case T_LONG: 172 result = (data->as_jlong()); 173 break; 174 default: 175 ShouldNotReachHere(); 176 result = 0; // unreachable 177 } 178 return result; 179 } 180 181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 182 Register base = addr->base()->as_pointer_register(); 183 LIR_Opr opr = addr->index(); 184 if (opr->is_cpu_register()) { 185 Register index; 186 if (opr->is_single_cpu()) 187 index = opr->as_register(); 188 else 189 index = opr->as_register_lo(); 190 assert(addr->disp() == 0, "must be"); 191 switch(opr->type()) { 192 case T_INT: 193 return Address(base, index, Address::sxtw(addr->scale())); 194 case T_LONG: 195 return Address(base, index, Address::lsl(addr->scale())); 196 default: 197 ShouldNotReachHere(); 198 } 199 } else { 200 intptr_t addr_offset = intptr_t(addr->disp()); 201 if (Address::offset_ok_for_immed(addr_offset, addr->scale())) 202 return Address(base, addr_offset, Address::lsl(addr->scale())); 203 else { 204 __ mov(tmp, addr_offset); 205 return Address(base, tmp, Address::lsl(addr->scale())); 206 } 207 } 208 return Address(); 209 } 210 211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 212 ShouldNotReachHere(); 213 return Address(); 214 } 215 216 Address LIR_Assembler::as_Address(LIR_Address* addr) { 217 return as_Address(addr, rscratch1); 218 } 219 220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 221 return as_Address(addr, rscratch1); // Ouch 222 // FIXME: This needs to be much more clever. See x86. 223 } 224 225 226 void LIR_Assembler::osr_entry() { 227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 228 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 229 ValueStack* entry_state = osr_entry->state(); 230 int number_of_locks = entry_state->locks_size(); 231 232 // we jump here if osr happens with the interpreter 233 // state set up to continue at the beginning of the 234 // loop that triggered osr - in particular, we have 235 // the following registers setup: 236 // 237 // r2: osr buffer 238 // 239 240 // build frame 241 ciMethod* m = compilation()->method(); 242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 243 244 // OSR buffer is 245 // 246 // locals[nlocals-1..0] 247 // monitors[0..number_of_locks] 248 // 249 // locals is a direct copy of the interpreter frame so in the osr buffer 250 // so first slot in the local array is the last local from the interpreter 251 // and last slot is local[0] (receiver) from the interpreter 252 // 253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 255 // in the interpreter frame (the method lock if a sync method) 256 257 // Initialize monitors in the compiled activation. 258 // r2: pointer to osr buffer 259 // 260 // All other registers are dead at this point and the locals will be 261 // copied into place by code emitted in the IR. 262 263 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 265 int monitor_offset = BytesPerWord * method()->max_locals() + 266 (2 * BytesPerWord) * (number_of_locks - 1); 267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 268 // the OSR buffer using 2 word entries: first the lock and then 269 // the oop. 270 for (int i = 0; i < number_of_locks; i++) { 271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 272 #ifdef ASSERT 273 // verify the interpreter's monitor has a non-null object 274 { 275 Label L; 276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 277 __ cbnz(rscratch1, L); 278 __ stop("locked object is NULL"); 279 __ bind(L); 280 } 281 #endif 282 __ ldr(r19, Address(OSR_buf, slot_offset + 0)); 283 __ str(r19, frame_map()->address_for_monitor_lock(i)); 284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 285 __ str(r19, frame_map()->address_for_monitor_object(i)); 286 } 287 } 288 } 289 290 291 // inline cache check; done before the frame is built. 292 int LIR_Assembler::check_icache() { 293 Register receiver = FrameMap::receiver_opr->as_register(); 294 Register ic_klass = IC_Klass; 295 int start_offset = __ offset(); 296 __ inline_cache_check(receiver, ic_klass); 297 298 // if icache check fails, then jump to runtime routine 299 // Note: RECEIVER must still contain the receiver! 300 Label dont; 301 __ br(Assembler::EQ, dont); 302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 303 304 // We align the verified entry point unless the method body 305 // (including its inline cache check) will fit in a single 64-byte 306 // icache line. 307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) { 308 // force alignment after the cache check. 309 __ align(CodeEntryAlignment); 310 } 311 312 __ bind(dont); 313 return start_offset; 314 } 315 316 317 void LIR_Assembler::jobject2reg(jobject o, Register reg) { 318 if (o == NULL) { 319 __ mov(reg, zr); 320 } else { 321 __ movoop(reg, o, /*immediate*/true); 322 } 323 } 324 325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) { 326 address target = NULL; 327 relocInfo::relocType reloc_type = relocInfo::none; 328 329 switch (patching_id(info)) { 330 case PatchingStub::access_field_id: 331 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 332 reloc_type = relocInfo::section_word_type; 333 break; 334 case PatchingStub::load_klass_id: 335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 336 reloc_type = relocInfo::metadata_type; 337 break; 338 case PatchingStub::load_mirror_id: 339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 340 reloc_type = relocInfo::oop_type; 341 break; 342 case PatchingStub::load_appendix_id: 343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 344 reloc_type = relocInfo::oop_type; 345 break; 346 default: ShouldNotReachHere(); 347 } 348 349 __ far_call(RuntimeAddress(target)); 350 add_call_info_here(info); 351 } 352 353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) { 354 deoptimize_trap(info); 355 } 356 357 358 // This specifies the rsp decrement needed to build the frame 359 int LIR_Assembler::initial_frame_size_in_bytes() const { 360 // if rounding, must let FrameMap know! 361 362 // The frame_map records size in slots (32bit word) 363 364 // subtract two words to account for return address and link 365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size; 366 } 367 368 369 int LIR_Assembler::emit_exception_handler() { 370 // if the last instruction is a call (typically to do a throw which 371 // is coming at the end after block reordering) the return address 372 // must still point into the code area in order to avoid assertion 373 // failures when searching for the corresponding bci => add a nop 374 // (was bug 5/14/1999 - gri) 375 __ nop(); 376 377 // generate code for exception handler 378 address handler_base = __ start_a_stub(exception_handler_size()); 379 if (handler_base == NULL) { 380 // not enough space left for the handler 381 bailout("exception handler overflow"); 382 return -1; 383 } 384 385 int offset = code_offset(); 386 387 // the exception oop and pc are in r0, and r3 388 // no other registers need to be preserved, so invalidate them 389 __ invalidate_registers(false, true, true, false, true, true); 390 391 // check that there is really an exception 392 __ verify_not_null_oop(r0); 393 394 // search an exception handler (r0: exception oop, r3: throwing pc) 395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here(); 396 guarantee(code_offset() - offset <= exception_handler_size(), "overflow"); 397 __ end_a_stub(); 398 399 return offset; 400 } 401 402 403 // Emit the code to remove the frame from the stack in the exception 404 // unwind path. 405 int LIR_Assembler::emit_unwind_handler() { 406 #ifndef PRODUCT 407 if (CommentedAssembly) { 408 _masm->block_comment("Unwind handler"); 409 } 410 #endif 411 412 int offset = code_offset(); 413 414 // Fetch the exception from TLS and clear out exception related thread state 415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset())); 416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset())); 417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset())); 418 419 __ bind(_unwind_handler_entry); 420 __ verify_not_null_oop(r0); 421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 422 __ mov(r19, r0); // Preserve the exception 423 } 424 425 // Preform needed unlocking 426 MonitorExitStub* stub = NULL; 427 if (method()->is_synchronized()) { 428 monitor_address(0, FrameMap::r0_opr); 429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0); 430 __ unlock_object(r5, r4, r0, *stub->entry()); 431 __ bind(*stub->continuation()); 432 } 433 434 if (compilation()->env()->dtrace_method_probes()) { 435 __ call_Unimplemented(); 436 #if 0 437 __ movptr(Address(rsp, 0), rax); 438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding()); 439 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit))); 440 #endif 441 } 442 443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 444 __ mov(r0, r19); // Restore the exception 445 } 446 447 // remove the activation and dispatch to the unwind handler 448 __ block_comment("remove_frame and dispatch to the unwind handler"); 449 __ remove_frame(initial_frame_size_in_bytes()); 450 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id))); 451 452 // Emit the slow path assembly 453 if (stub != NULL) { 454 stub->emit_code(this); 455 } 456 457 return offset; 458 } 459 460 461 int LIR_Assembler::emit_deopt_handler() { 462 // if the last instruction is a call (typically to do a throw which 463 // is coming at the end after block reordering) the return address 464 // must still point into the code area in order to avoid assertion 465 // failures when searching for the corresponding bci => add a nop 466 // (was bug 5/14/1999 - gri) 467 __ nop(); 468 469 // generate code for exception handler 470 address handler_base = __ start_a_stub(deopt_handler_size()); 471 if (handler_base == NULL) { 472 // not enough space left for the handler 473 bailout("deopt handler overflow"); 474 return -1; 475 } 476 477 int offset = code_offset(); 478 479 __ adr(lr, pc()); 480 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 481 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); 482 __ end_a_stub(); 483 484 return offset; 485 } 486 487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) { 488 _masm->code_section()->relocate(adr, relocInfo::poll_type); 489 int pc_offset = code_offset(); 490 flush_debug_info(pc_offset); 491 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset); 492 if (info->exception_handlers() != NULL) { 493 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers()); 494 } 495 } 496 497 void LIR_Assembler::return_op(LIR_Opr result) { 498 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,"); 499 500 // Pop the stack before the safepoint code 501 __ remove_frame(initial_frame_size_in_bytes()); 502 503 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) { 504 __ reserved_stack_check(); 505 } 506 507 address polling_page(os::get_polling_page()); 508 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type); 509 __ ret(lr); 510 } 511 512 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 513 address polling_page(os::get_polling_page()); 514 guarantee(info != NULL, "Shouldn't be NULL"); 515 assert(os::is_poll_address(polling_page), "should be"); 516 __ get_polling_page(rscratch1, polling_page, relocInfo::poll_type); 517 add_debug_info_for_branch(info); // This isn't just debug info: 518 // it's the oop map 519 __ read_polling_page(rscratch1, relocInfo::poll_type); 520 return __ offset(); 521 } 522 523 524 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 525 if (from_reg == r31_sp) 526 from_reg = sp; 527 if (to_reg == r31_sp) 528 to_reg = sp; 529 __ mov(to_reg, from_reg); 530 } 531 532 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); } 533 534 535 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 536 assert(src->is_constant(), "should not call otherwise"); 537 assert(dest->is_register(), "should not call otherwise"); 538 LIR_Const* c = src->as_constant_ptr(); 539 540 switch (c->type()) { 541 case T_INT: { 542 assert(patch_code == lir_patch_none, "no patching handled here"); 543 __ movw(dest->as_register(), c->as_jint()); 544 break; 545 } 546 547 case T_ADDRESS: { 548 assert(patch_code == lir_patch_none, "no patching handled here"); 549 __ mov(dest->as_register(), c->as_jint()); 550 break; 551 } 552 553 case T_LONG: { 554 assert(patch_code == lir_patch_none, "no patching handled here"); 555 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong()); 556 break; 557 } 558 559 case T_OBJECT: { 560 if (patch_code == lir_patch_none) { 561 jobject2reg(c->as_jobject(), dest->as_register()); 562 } else { 563 jobject2reg_with_patching(dest->as_register(), info); 564 } 565 break; 566 } 567 568 case T_METADATA: { 569 if (patch_code != lir_patch_none) { 570 klass2reg_with_patching(dest->as_register(), info); 571 } else { 572 __ mov_metadata(dest->as_register(), c->as_metadata()); 573 } 574 break; 575 } 576 577 case T_FLOAT: { 578 if (__ operand_valid_for_float_immediate(c->as_jfloat())) { 579 __ fmovs(dest->as_float_reg(), (c->as_jfloat())); 580 } else { 581 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat()))); 582 __ ldrs(dest->as_float_reg(), Address(rscratch1)); 583 } 584 break; 585 } 586 587 case T_DOUBLE: { 588 if (__ operand_valid_for_float_immediate(c->as_jdouble())) { 589 __ fmovd(dest->as_double_reg(), (c->as_jdouble())); 590 } else { 591 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble()))); 592 __ ldrd(dest->as_double_reg(), Address(rscratch1)); 593 } 594 break; 595 } 596 597 default: 598 ShouldNotReachHere(); 599 } 600 } 601 602 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 603 LIR_Const* c = src->as_constant_ptr(); 604 switch (c->type()) { 605 case T_OBJECT: 606 { 607 if (! c->as_jobject()) 608 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 609 else { 610 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL); 611 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 612 } 613 } 614 break; 615 case T_ADDRESS: 616 { 617 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL); 618 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 619 } 620 case T_INT: 621 case T_FLOAT: 622 { 623 Register reg = zr; 624 if (c->as_jint_bits() == 0) 625 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 626 else { 627 __ movw(rscratch1, c->as_jint_bits()); 628 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix())); 629 } 630 } 631 break; 632 case T_LONG: 633 case T_DOUBLE: 634 { 635 Register reg = zr; 636 if (c->as_jlong_bits() == 0) 637 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(), 638 lo_word_offset_in_bytes)); 639 else { 640 __ mov(rscratch1, (intptr_t)c->as_jlong_bits()); 641 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(), 642 lo_word_offset_in_bytes)); 643 } 644 } 645 break; 646 default: 647 ShouldNotReachHere(); 648 } 649 } 650 651 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 652 assert(src->is_constant(), "should not call otherwise"); 653 LIR_Const* c = src->as_constant_ptr(); 654 LIR_Address* to_addr = dest->as_address_ptr(); 655 656 void (Assembler::* insn)(Register Rt, const Address &adr); 657 658 switch (type) { 659 case T_ADDRESS: 660 assert(c->as_jint() == 0, "should be"); 661 insn = &Assembler::str; 662 break; 663 case T_LONG: 664 assert(c->as_jlong() == 0, "should be"); 665 insn = &Assembler::str; 666 break; 667 case T_INT: 668 assert(c->as_jint() == 0, "should be"); 669 insn = &Assembler::strw; 670 break; 671 case T_OBJECT: 672 case T_ARRAY: 673 assert(c->as_jobject() == 0, "should be"); 674 if (UseCompressedOops && !wide) { 675 insn = &Assembler::strw; 676 } else { 677 insn = &Assembler::str; 678 } 679 break; 680 case T_CHAR: 681 case T_SHORT: 682 assert(c->as_jint() == 0, "should be"); 683 insn = &Assembler::strh; 684 break; 685 case T_BOOLEAN: 686 case T_BYTE: 687 assert(c->as_jint() == 0, "should be"); 688 insn = &Assembler::strb; 689 break; 690 default: 691 ShouldNotReachHere(); 692 insn = &Assembler::str; // unreachable 693 } 694 695 if (info) add_debug_info_for_null_check_here(info); 696 (_masm->*insn)(zr, as_Address(to_addr, rscratch1)); 697 } 698 699 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 700 assert(src->is_register(), "should not call otherwise"); 701 assert(dest->is_register(), "should not call otherwise"); 702 703 // move between cpu-registers 704 if (dest->is_single_cpu()) { 705 if (src->type() == T_LONG) { 706 // Can do LONG -> OBJECT 707 move_regs(src->as_register_lo(), dest->as_register()); 708 return; 709 } 710 assert(src->is_single_cpu(), "must match"); 711 if (src->type() == T_OBJECT) { 712 __ verify_oop(src->as_register()); 713 } 714 move_regs(src->as_register(), dest->as_register()); 715 716 } else if (dest->is_double_cpu()) { 717 if (src->type() == T_OBJECT || src->type() == T_ARRAY) { 718 // Surprising to me but we can see move of a long to t_object 719 __ verify_oop(src->as_register()); 720 move_regs(src->as_register(), dest->as_register_lo()); 721 return; 722 } 723 assert(src->is_double_cpu(), "must match"); 724 Register f_lo = src->as_register_lo(); 725 Register f_hi = src->as_register_hi(); 726 Register t_lo = dest->as_register_lo(); 727 Register t_hi = dest->as_register_hi(); 728 assert(f_hi == f_lo, "must be same"); 729 assert(t_hi == t_lo, "must be same"); 730 move_regs(f_lo, t_lo); 731 732 } else if (dest->is_single_fpu()) { 733 __ fmovs(dest->as_float_reg(), src->as_float_reg()); 734 735 } else if (dest->is_double_fpu()) { 736 __ fmovd(dest->as_double_reg(), src->as_double_reg()); 737 738 } else { 739 ShouldNotReachHere(); 740 } 741 } 742 743 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 744 if (src->is_single_cpu()) { 745 if (type == T_ARRAY || type == T_OBJECT) { 746 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 747 __ verify_oop(src->as_register()); 748 } else if (type == T_METADATA || type == T_DOUBLE) { 749 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 750 } else { 751 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 752 } 753 754 } else if (src->is_double_cpu()) { 755 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes); 756 __ str(src->as_register_lo(), dest_addr_LO); 757 758 } else if (src->is_single_fpu()) { 759 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix()); 760 __ strs(src->as_float_reg(), dest_addr); 761 762 } else if (src->is_double_fpu()) { 763 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix()); 764 __ strd(src->as_double_reg(), dest_addr); 765 766 } else { 767 ShouldNotReachHere(); 768 } 769 770 } 771 772 773 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) { 774 LIR_Address* to_addr = dest->as_address_ptr(); 775 PatchingStub* patch = NULL; 776 Register compressed_src = rscratch1; 777 778 if (patch_code != lir_patch_none) { 779 deoptimize_trap(info); 780 return; 781 } 782 783 if (type == T_ARRAY || type == T_OBJECT) { 784 __ verify_oop(src->as_register()); 785 786 if (UseCompressedOops && !wide) { 787 __ encode_heap_oop(compressed_src, src->as_register()); 788 } else { 789 compressed_src = src->as_register(); 790 } 791 } 792 793 int null_check_here = code_offset(); 794 switch (type) { 795 case T_FLOAT: { 796 __ strs(src->as_float_reg(), as_Address(to_addr)); 797 break; 798 } 799 800 case T_DOUBLE: { 801 __ strd(src->as_double_reg(), as_Address(to_addr)); 802 break; 803 } 804 805 case T_ARRAY: // fall through 806 case T_OBJECT: // fall through 807 if (UseCompressedOops && !wide) { 808 __ strw(compressed_src, as_Address(to_addr, rscratch2)); 809 } else { 810 __ str(compressed_src, as_Address(to_addr)); 811 } 812 break; 813 case T_METADATA: 814 // We get here to store a method pointer to the stack to pass to 815 // a dtrace runtime call. This can't work on 64 bit with 816 // compressed klass ptrs: T_METADATA can be a compressed klass 817 // ptr or a 64 bit method pointer. 818 ShouldNotReachHere(); 819 __ str(src->as_register(), as_Address(to_addr)); 820 break; 821 case T_ADDRESS: 822 __ str(src->as_register(), as_Address(to_addr)); 823 break; 824 case T_INT: 825 __ strw(src->as_register(), as_Address(to_addr)); 826 break; 827 828 case T_LONG: { 829 __ str(src->as_register_lo(), as_Address_lo(to_addr)); 830 break; 831 } 832 833 case T_BYTE: // fall through 834 case T_BOOLEAN: { 835 __ strb(src->as_register(), as_Address(to_addr)); 836 break; 837 } 838 839 case T_CHAR: // fall through 840 case T_SHORT: 841 __ strh(src->as_register(), as_Address(to_addr)); 842 break; 843 844 default: 845 ShouldNotReachHere(); 846 } 847 if (info != NULL) { 848 add_debug_info_for_null_check(null_check_here, info); 849 } 850 } 851 852 853 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 854 assert(src->is_stack(), "should not call otherwise"); 855 assert(dest->is_register(), "should not call otherwise"); 856 857 if (dest->is_single_cpu()) { 858 if (type == T_ARRAY || type == T_OBJECT) { 859 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 860 __ verify_oop(dest->as_register()); 861 } else if (type == T_METADATA) { 862 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 863 } else { 864 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 865 } 866 867 } else if (dest->is_double_cpu()) { 868 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes); 869 __ ldr(dest->as_register_lo(), src_addr_LO); 870 871 } else if (dest->is_single_fpu()) { 872 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix()); 873 __ ldrs(dest->as_float_reg(), src_addr); 874 875 } else if (dest->is_double_fpu()) { 876 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix()); 877 __ ldrd(dest->as_double_reg(), src_addr); 878 879 } else { 880 ShouldNotReachHere(); 881 } 882 } 883 884 885 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 886 address target = NULL; 887 relocInfo::relocType reloc_type = relocInfo::none; 888 889 switch (patching_id(info)) { 890 case PatchingStub::access_field_id: 891 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 892 reloc_type = relocInfo::section_word_type; 893 break; 894 case PatchingStub::load_klass_id: 895 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 896 reloc_type = relocInfo::metadata_type; 897 break; 898 case PatchingStub::load_mirror_id: 899 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 900 reloc_type = relocInfo::oop_type; 901 break; 902 case PatchingStub::load_appendix_id: 903 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 904 reloc_type = relocInfo::oop_type; 905 break; 906 default: ShouldNotReachHere(); 907 } 908 909 __ far_call(RuntimeAddress(target)); 910 add_call_info_here(info); 911 } 912 913 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 914 915 LIR_Opr temp; 916 if (type == T_LONG || type == T_DOUBLE) 917 temp = FrameMap::rscratch1_long_opr; 918 else 919 temp = FrameMap::rscratch1_opr; 920 921 stack2reg(src, temp, src->type()); 922 reg2stack(temp, dest, dest->type(), false); 923 } 924 925 926 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) { 927 LIR_Address* addr = src->as_address_ptr(); 928 LIR_Address* from_addr = src->as_address_ptr(); 929 930 if (addr->base()->type() == T_OBJECT) { 931 __ verify_oop(addr->base()->as_pointer_register()); 932 } 933 934 if (patch_code != lir_patch_none) { 935 deoptimize_trap(info); 936 return; 937 } 938 939 if (info != NULL) { 940 add_debug_info_for_null_check_here(info); 941 } 942 int null_check_here = code_offset(); 943 switch (type) { 944 case T_FLOAT: { 945 __ ldrs(dest->as_float_reg(), as_Address(from_addr)); 946 break; 947 } 948 949 case T_DOUBLE: { 950 __ ldrd(dest->as_double_reg(), as_Address(from_addr)); 951 break; 952 } 953 954 case T_ARRAY: // fall through 955 case T_OBJECT: // fall through 956 if (UseCompressedOops && !wide) { 957 __ ldrw(dest->as_register(), as_Address(from_addr)); 958 } else { 959 __ ldr(dest->as_register(), as_Address(from_addr)); 960 } 961 break; 962 case T_METADATA: 963 // We get here to store a method pointer to the stack to pass to 964 // a dtrace runtime call. This can't work on 64 bit with 965 // compressed klass ptrs: T_METADATA can be a compressed klass 966 // ptr or a 64 bit method pointer. 967 ShouldNotReachHere(); 968 __ ldr(dest->as_register(), as_Address(from_addr)); 969 break; 970 case T_ADDRESS: 971 // FIXME: OMG this is a horrible kludge. Any offset from an 972 // address that matches klass_offset_in_bytes() will be loaded 973 // as a word, not a long. 974 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { 975 __ ldrw(dest->as_register(), as_Address(from_addr)); 976 } else { 977 __ ldr(dest->as_register(), as_Address(from_addr)); 978 } 979 break; 980 case T_INT: 981 __ ldrw(dest->as_register(), as_Address(from_addr)); 982 break; 983 984 case T_LONG: { 985 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr)); 986 break; 987 } 988 989 case T_BYTE: 990 __ ldrsb(dest->as_register(), as_Address(from_addr)); 991 break; 992 case T_BOOLEAN: { 993 __ ldrb(dest->as_register(), as_Address(from_addr)); 994 break; 995 } 996 997 case T_CHAR: 998 __ ldrh(dest->as_register(), as_Address(from_addr)); 999 break; 1000 case T_SHORT: 1001 __ ldrsh(dest->as_register(), as_Address(from_addr)); 1002 break; 1003 1004 default: 1005 ShouldNotReachHere(); 1006 } 1007 1008 if (type == T_ARRAY || type == T_OBJECT) { 1009 if (UseCompressedOops && !wide) { 1010 __ decode_heap_oop(dest->as_register()); 1011 } 1012 __ verify_oop(dest->as_register()); 1013 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1014 if (UseCompressedClassPointers) { 1015 __ decode_klass_not_null(dest->as_register()); 1016 } 1017 } 1018 } 1019 1020 1021 int LIR_Assembler::array_element_size(BasicType type) const { 1022 int elem_size = type2aelembytes(type); 1023 return exact_log2(elem_size); 1024 } 1025 1026 void LIR_Assembler::arithmetic_idiv(LIR_Op3* op, bool is_irem) { 1027 Register Rdividend = op->in_opr1()->as_register(); 1028 Register Rdivisor = op->in_opr2()->as_register(); 1029 Register Rscratch = op->in_opr3()->as_register(); 1030 Register Rresult = op->result_opr()->as_register(); 1031 int divisor = -1; 1032 1033 /* 1034 TODO: For some reason, using the Rscratch that gets passed in is 1035 not possible because the register allocator does not see the tmp reg 1036 as used, and assignes it the same register as Rdividend. We use rscratch1 1037 instead. 1038 1039 assert(Rdividend != Rscratch, ""); 1040 assert(Rdivisor != Rscratch, ""); 1041 */ 1042 1043 if (Rdivisor == noreg && is_power_of_2(divisor)) { 1044 // convert division by a power of two into some shifts and logical operations 1045 } 1046 1047 __ corrected_idivl(Rresult, Rdividend, Rdivisor, is_irem, rscratch1); 1048 } 1049 1050 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1051 switch (op->code()) { 1052 case lir_idiv: 1053 arithmetic_idiv(op, false); 1054 break; 1055 case lir_irem: 1056 arithmetic_idiv(op, true); 1057 break; 1058 case lir_fmad: 1059 __ fmaddd(op->result_opr()->as_double_reg(), 1060 op->in_opr1()->as_double_reg(), 1061 op->in_opr2()->as_double_reg(), 1062 op->in_opr3()->as_double_reg()); 1063 break; 1064 case lir_fmaf: 1065 __ fmadds(op->result_opr()->as_float_reg(), 1066 op->in_opr1()->as_float_reg(), 1067 op->in_opr2()->as_float_reg(), 1068 op->in_opr3()->as_float_reg()); 1069 break; 1070 default: ShouldNotReachHere(); break; 1071 } 1072 } 1073 1074 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1075 #ifdef ASSERT 1076 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label"); 1077 if (op->block() != NULL) _branch_target_blocks.append(op->block()); 1078 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock()); 1079 #endif 1080 1081 if (op->cond() == lir_cond_always) { 1082 if (op->info() != NULL) add_debug_info_for_branch(op->info()); 1083 __ b(*(op->label())); 1084 } else { 1085 Assembler::Condition acond; 1086 if (op->code() == lir_cond_float_branch) { 1087 bool is_unordered = (op->ublock() == op->block()); 1088 // Assembler::EQ does not permit unordered branches, so we add 1089 // another branch here. Likewise, Assembler::NE does not permit 1090 // ordered branches. 1091 if (is_unordered && op->cond() == lir_cond_equal 1092 || !is_unordered && op->cond() == lir_cond_notEqual) 1093 __ br(Assembler::VS, *(op->ublock()->label())); 1094 switch(op->cond()) { 1095 case lir_cond_equal: acond = Assembler::EQ; break; 1096 case lir_cond_notEqual: acond = Assembler::NE; break; 1097 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break; 1098 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break; 1099 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break; 1100 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break; 1101 default: ShouldNotReachHere(); 1102 acond = Assembler::EQ; // unreachable 1103 } 1104 } else { 1105 switch (op->cond()) { 1106 case lir_cond_equal: acond = Assembler::EQ; break; 1107 case lir_cond_notEqual: acond = Assembler::NE; break; 1108 case lir_cond_less: acond = Assembler::LT; break; 1109 case lir_cond_lessEqual: acond = Assembler::LE; break; 1110 case lir_cond_greaterEqual: acond = Assembler::GE; break; 1111 case lir_cond_greater: acond = Assembler::GT; break; 1112 case lir_cond_belowEqual: acond = Assembler::LS; break; 1113 case lir_cond_aboveEqual: acond = Assembler::HS; break; 1114 default: ShouldNotReachHere(); 1115 acond = Assembler::EQ; // unreachable 1116 } 1117 } 1118 __ br(acond,*(op->label())); 1119 } 1120 } 1121 1122 1123 1124 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1125 LIR_Opr src = op->in_opr(); 1126 LIR_Opr dest = op->result_opr(); 1127 1128 switch (op->bytecode()) { 1129 case Bytecodes::_i2f: 1130 { 1131 __ scvtfws(dest->as_float_reg(), src->as_register()); 1132 break; 1133 } 1134 case Bytecodes::_i2d: 1135 { 1136 __ scvtfwd(dest->as_double_reg(), src->as_register()); 1137 break; 1138 } 1139 case Bytecodes::_l2d: 1140 { 1141 __ scvtfd(dest->as_double_reg(), src->as_register_lo()); 1142 break; 1143 } 1144 case Bytecodes::_l2f: 1145 { 1146 __ scvtfs(dest->as_float_reg(), src->as_register_lo()); 1147 break; 1148 } 1149 case Bytecodes::_f2d: 1150 { 1151 __ fcvts(dest->as_double_reg(), src->as_float_reg()); 1152 break; 1153 } 1154 case Bytecodes::_d2f: 1155 { 1156 __ fcvtd(dest->as_float_reg(), src->as_double_reg()); 1157 break; 1158 } 1159 case Bytecodes::_i2c: 1160 { 1161 __ ubfx(dest->as_register(), src->as_register(), 0, 16); 1162 break; 1163 } 1164 case Bytecodes::_i2l: 1165 { 1166 __ sxtw(dest->as_register_lo(), src->as_register()); 1167 break; 1168 } 1169 case Bytecodes::_i2s: 1170 { 1171 __ sxth(dest->as_register(), src->as_register()); 1172 break; 1173 } 1174 case Bytecodes::_i2b: 1175 { 1176 __ sxtb(dest->as_register(), src->as_register()); 1177 break; 1178 } 1179 case Bytecodes::_l2i: 1180 { 1181 _masm->block_comment("FIXME: This could be a no-op"); 1182 __ uxtw(dest->as_register(), src->as_register_lo()); 1183 break; 1184 } 1185 case Bytecodes::_d2l: 1186 { 1187 __ fcvtzd(dest->as_register_lo(), src->as_double_reg()); 1188 break; 1189 } 1190 case Bytecodes::_f2i: 1191 { 1192 __ fcvtzsw(dest->as_register(), src->as_float_reg()); 1193 break; 1194 } 1195 case Bytecodes::_f2l: 1196 { 1197 __ fcvtzs(dest->as_register_lo(), src->as_float_reg()); 1198 break; 1199 } 1200 case Bytecodes::_d2i: 1201 { 1202 __ fcvtzdw(dest->as_register(), src->as_double_reg()); 1203 break; 1204 } 1205 default: ShouldNotReachHere(); 1206 } 1207 } 1208 1209 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1210 if (op->init_check()) { 1211 __ ldrb(rscratch1, Address(op->klass()->as_register(), 1212 InstanceKlass::init_state_offset())); 1213 __ cmpw(rscratch1, InstanceKlass::fully_initialized); 1214 add_debug_info_for_null_check_here(op->stub()->info()); 1215 __ br(Assembler::NE, *op->stub()->entry()); 1216 } 1217 __ allocate_object(op->obj()->as_register(), 1218 op->tmp1()->as_register(), 1219 op->tmp2()->as_register(), 1220 op->header_size(), 1221 op->object_size(), 1222 op->klass()->as_register(), 1223 *op->stub()->entry()); 1224 __ bind(*op->stub()->continuation()); 1225 } 1226 1227 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1228 Register len = op->len()->as_register(); 1229 __ uxtw(len, len); 1230 1231 if (UseSlowPath || 1232 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || 1233 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { 1234 __ b(*op->stub()->entry()); 1235 } else { 1236 Register tmp1 = op->tmp1()->as_register(); 1237 Register tmp2 = op->tmp2()->as_register(); 1238 Register tmp3 = op->tmp3()->as_register(); 1239 if (len == tmp1) { 1240 tmp1 = tmp3; 1241 } else if (len == tmp2) { 1242 tmp2 = tmp3; 1243 } else if (len == tmp3) { 1244 // everything is ok 1245 } else { 1246 __ mov(tmp3, len); 1247 } 1248 __ allocate_array(op->obj()->as_register(), 1249 len, 1250 tmp1, 1251 tmp2, 1252 arrayOopDesc::header_size(op->type()), 1253 array_element_size(op->type()), 1254 op->klass()->as_register(), 1255 *op->stub()->entry()); 1256 } 1257 __ bind(*op->stub()->continuation()); 1258 } 1259 1260 void LIR_Assembler::type_profile_helper(Register mdo, 1261 ciMethodData *md, ciProfileData *data, 1262 Register recv, Label* update_done) { 1263 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1264 Label next_test; 1265 // See if the receiver is receiver[n]. 1266 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1267 __ ldr(rscratch1, Address(rscratch2)); 1268 __ cmp(recv, rscratch1); 1269 __ br(Assembler::NE, next_test); 1270 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); 1271 __ addptr(data_addr, DataLayout::counter_increment); 1272 __ b(*update_done); 1273 __ bind(next_test); 1274 } 1275 1276 // Didn't find receiver; find next empty slot and fill it in 1277 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1278 Label next_test; 1279 __ lea(rscratch2, 1280 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1281 Address recv_addr(rscratch2); 1282 __ ldr(rscratch1, recv_addr); 1283 __ cbnz(rscratch1, next_test); 1284 __ str(recv, recv_addr); 1285 __ mov(rscratch1, DataLayout::counter_increment); 1286 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)))); 1287 __ str(rscratch1, Address(rscratch2)); 1288 __ b(*update_done); 1289 __ bind(next_test); 1290 } 1291 } 1292 1293 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1294 // we always need a stub for the failure case. 1295 CodeStub* stub = op->stub(); 1296 Register obj = op->object()->as_register(); 1297 Register k_RInfo = op->tmp1()->as_register(); 1298 Register klass_RInfo = op->tmp2()->as_register(); 1299 Register dst = op->result_opr()->as_register(); 1300 ciKlass* k = op->klass(); 1301 Register Rtmp1 = noreg; 1302 1303 // check if it needs to be profiled 1304 ciMethodData* md; 1305 ciProfileData* data; 1306 1307 const bool should_profile = op->should_profile(); 1308 1309 if (should_profile) { 1310 ciMethod* method = op->profiled_method(); 1311 assert(method != NULL, "Should have method"); 1312 int bci = op->profiled_bci(); 1313 md = method->method_data_or_null(); 1314 assert(md != NULL, "Sanity"); 1315 data = md->bci_to_data(bci); 1316 assert(data != NULL, "need data for type check"); 1317 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1318 } 1319 Label profile_cast_success, profile_cast_failure; 1320 Label *success_target = should_profile ? &profile_cast_success : success; 1321 Label *failure_target = should_profile ? &profile_cast_failure : failure; 1322 1323 if (obj == k_RInfo) { 1324 k_RInfo = dst; 1325 } else if (obj == klass_RInfo) { 1326 klass_RInfo = dst; 1327 } 1328 if (k->is_loaded() && !UseCompressedClassPointers) { 1329 select_different_registers(obj, dst, k_RInfo, klass_RInfo); 1330 } else { 1331 Rtmp1 = op->tmp3()->as_register(); 1332 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1333 } 1334 1335 assert_different_registers(obj, k_RInfo, klass_RInfo); 1336 1337 if (should_profile) { 1338 Label not_null; 1339 __ cbnz(obj, not_null); 1340 // Object is null; update MDO and exit 1341 Register mdo = klass_RInfo; 1342 __ mov_metadata(mdo, md->constant_encoding()); 1343 Address data_addr 1344 = __ form_address(rscratch2, mdo, 1345 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()), 1346 LogBytesPerWord); 1347 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant()); 1348 __ ldr(rscratch1, data_addr); 1349 __ orr(rscratch1, rscratch1, header_bits); 1350 __ str(rscratch1, data_addr); 1351 __ b(*obj_is_null); 1352 __ bind(not_null); 1353 } else { 1354 __ cbz(obj, *obj_is_null); 1355 } 1356 1357 if (!k->is_loaded()) { 1358 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1359 } else { 1360 __ mov_metadata(k_RInfo, k->constant_encoding()); 1361 } 1362 __ verify_oop(obj); 1363 1364 if (op->fast_check()) { 1365 // get object class 1366 // not a safepoint as obj null check happens earlier 1367 __ load_klass(rscratch1, obj); 1368 __ cmp( rscratch1, k_RInfo); 1369 1370 __ br(Assembler::NE, *failure_target); 1371 // successful cast, fall through to profile or jump 1372 } else { 1373 // get object class 1374 // not a safepoint as obj null check happens earlier 1375 __ load_klass(klass_RInfo, obj); 1376 if (k->is_loaded()) { 1377 // See if we get an immediate positive hit 1378 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset()))); 1379 __ cmp(k_RInfo, rscratch1); 1380 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1381 __ br(Assembler::NE, *failure_target); 1382 // successful cast, fall through to profile or jump 1383 } else { 1384 // See if we get an immediate positive hit 1385 __ br(Assembler::EQ, *success_target); 1386 // check for self 1387 __ cmp(klass_RInfo, k_RInfo); 1388 __ br(Assembler::EQ, *success_target); 1389 1390 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1391 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1392 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1393 // result is a boolean 1394 __ cbzw(klass_RInfo, *failure_target); 1395 // successful cast, fall through to profile or jump 1396 } 1397 } else { 1398 // perform the fast part of the checking logic 1399 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1400 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1401 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1402 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1403 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1404 // result is a boolean 1405 __ cbz(k_RInfo, *failure_target); 1406 // successful cast, fall through to profile or jump 1407 } 1408 } 1409 if (should_profile) { 1410 Register mdo = klass_RInfo, recv = k_RInfo; 1411 __ bind(profile_cast_success); 1412 __ mov_metadata(mdo, md->constant_encoding()); 1413 __ load_klass(recv, obj); 1414 Label update_done; 1415 type_profile_helper(mdo, md, data, recv, success); 1416 __ b(*success); 1417 1418 __ bind(profile_cast_failure); 1419 __ mov_metadata(mdo, md->constant_encoding()); 1420 Address counter_addr 1421 = __ form_address(rscratch2, mdo, 1422 md->byte_offset_of_slot(data, CounterData::count_offset()), 1423 LogBytesPerWord); 1424 __ ldr(rscratch1, counter_addr); 1425 __ sub(rscratch1, rscratch1, DataLayout::counter_increment); 1426 __ str(rscratch1, counter_addr); 1427 __ b(*failure); 1428 } 1429 __ b(*success); 1430 } 1431 1432 1433 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1434 const bool should_profile = op->should_profile(); 1435 1436 LIR_Code code = op->code(); 1437 if (code == lir_store_check) { 1438 Register value = op->object()->as_register(); 1439 Register array = op->array()->as_register(); 1440 Register k_RInfo = op->tmp1()->as_register(); 1441 Register klass_RInfo = op->tmp2()->as_register(); 1442 Register Rtmp1 = op->tmp3()->as_register(); 1443 1444 CodeStub* stub = op->stub(); 1445 1446 // check if it needs to be profiled 1447 ciMethodData* md; 1448 ciProfileData* data; 1449 1450 if (should_profile) { 1451 ciMethod* method = op->profiled_method(); 1452 assert(method != NULL, "Should have method"); 1453 int bci = op->profiled_bci(); 1454 md = method->method_data_or_null(); 1455 assert(md != NULL, "Sanity"); 1456 data = md->bci_to_data(bci); 1457 assert(data != NULL, "need data for type check"); 1458 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1459 } 1460 Label profile_cast_success, profile_cast_failure, done; 1461 Label *success_target = should_profile ? &profile_cast_success : &done; 1462 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry(); 1463 1464 if (should_profile) { 1465 Label not_null; 1466 __ cbnz(value, not_null); 1467 // Object is null; update MDO and exit 1468 Register mdo = klass_RInfo; 1469 __ mov_metadata(mdo, md->constant_encoding()); 1470 Address data_addr 1471 = __ form_address(rscratch2, mdo, 1472 md->byte_offset_of_slot(data, DataLayout::header_offset()), 1473 LogBytesPerInt); 1474 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant()); 1475 __ ldrw(rscratch1, data_addr); 1476 __ orrw(rscratch1, rscratch1, header_bits); 1477 __ strw(rscratch1, data_addr); 1478 __ b(done); 1479 __ bind(not_null); 1480 } else { 1481 __ cbz(value, done); 1482 } 1483 1484 add_debug_info_for_null_check_here(op->info_for_exception()); 1485 __ load_klass(k_RInfo, array); 1486 __ load_klass(klass_RInfo, value); 1487 1488 // get instance klass (it's already uncompressed) 1489 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1490 // perform the fast part of the checking logic 1491 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1492 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1493 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1494 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1495 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1496 // result is a boolean 1497 __ cbzw(k_RInfo, *failure_target); 1498 // fall through to the success case 1499 1500 if (should_profile) { 1501 Register mdo = klass_RInfo, recv = k_RInfo; 1502 __ bind(profile_cast_success); 1503 __ mov_metadata(mdo, md->constant_encoding()); 1504 __ load_klass(recv, value); 1505 Label update_done; 1506 type_profile_helper(mdo, md, data, recv, &done); 1507 __ b(done); 1508 1509 __ bind(profile_cast_failure); 1510 __ mov_metadata(mdo, md->constant_encoding()); 1511 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1512 __ lea(rscratch2, counter_addr); 1513 __ ldr(rscratch1, Address(rscratch2)); 1514 __ sub(rscratch1, rscratch1, DataLayout::counter_increment); 1515 __ str(rscratch1, Address(rscratch2)); 1516 __ b(*stub->entry()); 1517 } 1518 1519 __ bind(done); 1520 } else if (code == lir_checkcast) { 1521 Register obj = op->object()->as_register(); 1522 Register dst = op->result_opr()->as_register(); 1523 Label success; 1524 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1525 __ bind(success); 1526 if (dst != obj) { 1527 __ mov(dst, obj); 1528 } 1529 } else if (code == lir_instanceof) { 1530 Register obj = op->object()->as_register(); 1531 Register dst = op->result_opr()->as_register(); 1532 Label success, failure, done; 1533 emit_typecheck_helper(op, &success, &failure, &failure); 1534 __ bind(failure); 1535 __ mov(dst, zr); 1536 __ b(done); 1537 __ bind(success); 1538 __ mov(dst, 1); 1539 __ bind(done); 1540 } else { 1541 ShouldNotReachHere(); 1542 } 1543 } 1544 1545 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { 1546 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); 1547 __ cset(rscratch1, Assembler::NE); 1548 __ membar(__ AnyAny); 1549 } 1550 1551 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { 1552 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); 1553 __ cset(rscratch1, Assembler::NE); 1554 __ membar(__ AnyAny); 1555 } 1556 1557 1558 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1559 assert(VM_Version::supports_cx8(), "wrong machine"); 1560 Register addr = as_reg(op->addr()); 1561 Register newval = as_reg(op->new_value()); 1562 Register cmpval = as_reg(op->cmp_value()); 1563 Label succeed, fail, around; 1564 1565 if (op->code() == lir_cas_obj) { 1566 if (UseCompressedOops) { 1567 Register t1 = op->tmp1()->as_register(); 1568 assert(op->tmp1()->is_valid(), "must be"); 1569 __ encode_heap_oop(t1, cmpval); 1570 cmpval = t1; 1571 __ encode_heap_oop(rscratch2, newval); 1572 newval = rscratch2; 1573 casw(addr, newval, cmpval); 1574 } else { 1575 casl(addr, newval, cmpval); 1576 } 1577 } else if (op->code() == lir_cas_int) { 1578 casw(addr, newval, cmpval); 1579 } else { 1580 casl(addr, newval, cmpval); 1581 } 1582 } 1583 1584 1585 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) { 1586 1587 Assembler::Condition acond, ncond; 1588 switch (condition) { 1589 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break; 1590 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break; 1591 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break; 1592 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break; 1593 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break; 1594 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break; 1595 case lir_cond_belowEqual: 1596 case lir_cond_aboveEqual: 1597 default: ShouldNotReachHere(); 1598 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable 1599 } 1600 1601 assert(result->is_single_cpu() || result->is_double_cpu(), 1602 "expect single register for result"); 1603 if (opr1->is_constant() && opr2->is_constant() 1604 && opr1->type() == T_INT && opr2->type() == T_INT) { 1605 jint val1 = opr1->as_jint(); 1606 jint val2 = opr2->as_jint(); 1607 if (val1 == 0 && val2 == 1) { 1608 __ cset(result->as_register(), ncond); 1609 return; 1610 } else if (val1 == 1 && val2 == 0) { 1611 __ cset(result->as_register(), acond); 1612 return; 1613 } 1614 } 1615 1616 if (opr1->is_constant() && opr2->is_constant() 1617 && opr1->type() == T_LONG && opr2->type() == T_LONG) { 1618 jlong val1 = opr1->as_jlong(); 1619 jlong val2 = opr2->as_jlong(); 1620 if (val1 == 0 && val2 == 1) { 1621 __ cset(result->as_register_lo(), ncond); 1622 return; 1623 } else if (val1 == 1 && val2 == 0) { 1624 __ cset(result->as_register_lo(), acond); 1625 return; 1626 } 1627 } 1628 1629 if (opr1->is_stack()) { 1630 stack2reg(opr1, FrameMap::rscratch1_opr, result->type()); 1631 opr1 = FrameMap::rscratch1_opr; 1632 } else if (opr1->is_constant()) { 1633 LIR_Opr tmp 1634 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr; 1635 const2reg(opr1, tmp, lir_patch_none, NULL); 1636 opr1 = tmp; 1637 } 1638 1639 if (opr2->is_stack()) { 1640 stack2reg(opr2, FrameMap::rscratch2_opr, result->type()); 1641 opr2 = FrameMap::rscratch2_opr; 1642 } else if (opr2->is_constant()) { 1643 LIR_Opr tmp 1644 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr; 1645 const2reg(opr2, tmp, lir_patch_none, NULL); 1646 opr2 = tmp; 1647 } 1648 1649 if (result->type() == T_LONG) 1650 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond); 1651 else 1652 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond); 1653 } 1654 1655 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 1656 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method"); 1657 1658 if (left->is_single_cpu()) { 1659 Register lreg = left->as_register(); 1660 Register dreg = as_reg(dest); 1661 1662 if (right->is_single_cpu()) { 1663 // cpu register - cpu register 1664 1665 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT, 1666 "should be"); 1667 Register rreg = right->as_register(); 1668 switch (code) { 1669 case lir_add: __ addw (dest->as_register(), lreg, rreg); break; 1670 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break; 1671 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break; 1672 default: ShouldNotReachHere(); 1673 } 1674 1675 } else if (right->is_double_cpu()) { 1676 Register rreg = right->as_register_lo(); 1677 // single_cpu + double_cpu: can happen with obj+long 1678 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1679 switch (code) { 1680 case lir_add: __ add(dreg, lreg, rreg); break; 1681 case lir_sub: __ sub(dreg, lreg, rreg); break; 1682 default: ShouldNotReachHere(); 1683 } 1684 } else if (right->is_constant()) { 1685 // cpu register - constant 1686 jlong c; 1687 1688 // FIXME. This is fugly: we really need to factor all this logic. 1689 switch(right->type()) { 1690 case T_LONG: 1691 c = right->as_constant_ptr()->as_jlong(); 1692 break; 1693 case T_INT: 1694 case T_ADDRESS: 1695 c = right->as_constant_ptr()->as_jint(); 1696 break; 1697 default: 1698 ShouldNotReachHere(); 1699 c = 0; // unreachable 1700 break; 1701 } 1702 1703 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1704 if (c == 0 && dreg == lreg) { 1705 COMMENT("effective nop elided"); 1706 return; 1707 } 1708 switch(left->type()) { 1709 case T_INT: 1710 switch (code) { 1711 case lir_add: __ addw(dreg, lreg, c); break; 1712 case lir_sub: __ subw(dreg, lreg, c); break; 1713 default: ShouldNotReachHere(); 1714 } 1715 break; 1716 case T_OBJECT: 1717 case T_ADDRESS: 1718 switch (code) { 1719 case lir_add: __ add(dreg, lreg, c); break; 1720 case lir_sub: __ sub(dreg, lreg, c); break; 1721 default: ShouldNotReachHere(); 1722 } 1723 break; 1724 ShouldNotReachHere(); 1725 } 1726 } else { 1727 ShouldNotReachHere(); 1728 } 1729 1730 } else if (left->is_double_cpu()) { 1731 Register lreg_lo = left->as_register_lo(); 1732 1733 if (right->is_double_cpu()) { 1734 // cpu register - cpu register 1735 Register rreg_lo = right->as_register_lo(); 1736 switch (code) { 1737 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1738 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1739 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1740 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break; 1741 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break; 1742 default: 1743 ShouldNotReachHere(); 1744 } 1745 1746 } else if (right->is_constant()) { 1747 jlong c = right->as_constant_ptr()->as_jlong_bits(); 1748 Register dreg = as_reg(dest); 1749 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1750 if (c == 0 && dreg == lreg_lo) { 1751 COMMENT("effective nop elided"); 1752 return; 1753 } 1754 switch (code) { 1755 case lir_add: __ add(dreg, lreg_lo, c); break; 1756 case lir_sub: __ sub(dreg, lreg_lo, c); break; 1757 default: 1758 ShouldNotReachHere(); 1759 } 1760 } else { 1761 ShouldNotReachHere(); 1762 } 1763 } else if (left->is_single_fpu()) { 1764 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register"); 1765 switch (code) { 1766 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1767 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1768 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1769 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1770 default: 1771 ShouldNotReachHere(); 1772 } 1773 } else if (left->is_double_fpu()) { 1774 if (right->is_double_fpu()) { 1775 // cpu register - cpu register 1776 switch (code) { 1777 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1778 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1779 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1780 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1781 default: 1782 ShouldNotReachHere(); 1783 } 1784 } else { 1785 if (right->is_constant()) { 1786 ShouldNotReachHere(); 1787 } 1788 ShouldNotReachHere(); 1789 } 1790 } else if (left->is_single_stack() || left->is_address()) { 1791 assert(left == dest, "left and dest must be equal"); 1792 ShouldNotReachHere(); 1793 } else { 1794 ShouldNotReachHere(); 1795 } 1796 } 1797 1798 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); } 1799 1800 1801 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { 1802 switch(code) { 1803 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break; 1804 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break; 1805 default : ShouldNotReachHere(); 1806 } 1807 } 1808 1809 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 1810 1811 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register"); 1812 Register Rleft = left->is_single_cpu() ? left->as_register() : 1813 left->as_register_lo(); 1814 if (dst->is_single_cpu()) { 1815 Register Rdst = dst->as_register(); 1816 if (right->is_constant()) { 1817 switch (code) { 1818 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break; 1819 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break; 1820 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break; 1821 default: ShouldNotReachHere(); break; 1822 } 1823 } else { 1824 Register Rright = right->is_single_cpu() ? right->as_register() : 1825 right->as_register_lo(); 1826 switch (code) { 1827 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break; 1828 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break; 1829 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break; 1830 default: ShouldNotReachHere(); break; 1831 } 1832 } 1833 } else { 1834 Register Rdst = dst->as_register_lo(); 1835 if (right->is_constant()) { 1836 switch (code) { 1837 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break; 1838 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break; 1839 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break; 1840 default: ShouldNotReachHere(); break; 1841 } 1842 } else { 1843 Register Rright = right->is_single_cpu() ? right->as_register() : 1844 right->as_register_lo(); 1845 switch (code) { 1846 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break; 1847 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break; 1848 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break; 1849 default: ShouldNotReachHere(); break; 1850 } 1851 } 1852 } 1853 } 1854 1855 1856 1857 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); } 1858 1859 1860 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 1861 if (opr1->is_constant() && opr2->is_single_cpu()) { 1862 // tableswitch 1863 Register reg = as_reg(opr2); 1864 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()]; 1865 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after); 1866 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) { 1867 Register reg1 = as_reg(opr1); 1868 if (opr2->is_single_cpu()) { 1869 // cpu register - cpu register 1870 Register reg2 = opr2->as_register(); 1871 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) { 1872 __ cmp(reg1, reg2); 1873 } else { 1874 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?"); 1875 __ cmpw(reg1, reg2); 1876 } 1877 return; 1878 } 1879 if (opr2->is_double_cpu()) { 1880 // cpu register - cpu register 1881 Register reg2 = opr2->as_register_lo(); 1882 __ cmp(reg1, reg2); 1883 return; 1884 } 1885 1886 if (opr2->is_constant()) { 1887 bool is_32bit = false; // width of register operand 1888 jlong imm; 1889 1890 switch(opr2->type()) { 1891 case T_INT: 1892 imm = opr2->as_constant_ptr()->as_jint(); 1893 is_32bit = true; 1894 break; 1895 case T_LONG: 1896 imm = opr2->as_constant_ptr()->as_jlong(); 1897 break; 1898 case T_ADDRESS: 1899 imm = opr2->as_constant_ptr()->as_jint(); 1900 break; 1901 case T_OBJECT: 1902 case T_ARRAY: 1903 imm = jlong(opr2->as_constant_ptr()->as_jobject()); 1904 break; 1905 default: 1906 ShouldNotReachHere(); 1907 imm = 0; // unreachable 1908 break; 1909 } 1910 1911 if (Assembler::operand_valid_for_add_sub_immediate(imm)) { 1912 if (is_32bit) 1913 __ cmpw(reg1, imm); 1914 else 1915 __ cmp(reg1, imm); 1916 return; 1917 } else { 1918 __ mov(rscratch1, imm); 1919 if (is_32bit) 1920 __ cmpw(reg1, rscratch1); 1921 else 1922 __ cmp(reg1, rscratch1); 1923 return; 1924 } 1925 } else 1926 ShouldNotReachHere(); 1927 } else if (opr1->is_single_fpu()) { 1928 FloatRegister reg1 = opr1->as_float_reg(); 1929 assert(opr2->is_single_fpu(), "expect single float register"); 1930 FloatRegister reg2 = opr2->as_float_reg(); 1931 __ fcmps(reg1, reg2); 1932 } else if (opr1->is_double_fpu()) { 1933 FloatRegister reg1 = opr1->as_double_reg(); 1934 assert(opr2->is_double_fpu(), "expect double float register"); 1935 FloatRegister reg2 = opr2->as_double_reg(); 1936 __ fcmpd(reg1, reg2); 1937 } else { 1938 ShouldNotReachHere(); 1939 } 1940 } 1941 1942 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){ 1943 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 1944 bool is_unordered_less = (code == lir_ucmp_fd2i); 1945 if (left->is_single_fpu()) { 1946 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register()); 1947 } else if (left->is_double_fpu()) { 1948 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register()); 1949 } else { 1950 ShouldNotReachHere(); 1951 } 1952 } else if (code == lir_cmp_l2i) { 1953 Label done; 1954 __ cmp(left->as_register_lo(), right->as_register_lo()); 1955 __ mov(dst->as_register(), (u_int64_t)-1L); 1956 __ br(Assembler::LT, done); 1957 __ csinc(dst->as_register(), zr, zr, Assembler::EQ); 1958 __ bind(done); 1959 } else { 1960 ShouldNotReachHere(); 1961 } 1962 } 1963 1964 1965 void LIR_Assembler::align_call(LIR_Code code) { } 1966 1967 1968 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 1969 address call = __ trampoline_call(Address(op->addr(), rtype)); 1970 if (call == NULL) { 1971 bailout("trampoline stub overflow"); 1972 return; 1973 } 1974 add_call_info(code_offset(), op->info()); 1975 } 1976 1977 1978 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 1979 address call = __ ic_call(op->addr()); 1980 if (call == NULL) { 1981 bailout("trampoline stub overflow"); 1982 return; 1983 } 1984 add_call_info(code_offset(), op->info()); 1985 } 1986 1987 1988 /* Currently, vtable-dispatch is only enabled for sparc platforms */ 1989 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) { 1990 ShouldNotReachHere(); 1991 } 1992 1993 1994 void LIR_Assembler::emit_static_call_stub() { 1995 address call_pc = __ pc(); 1996 address stub = __ start_a_stub(call_stub_size()); 1997 if (stub == NULL) { 1998 bailout("static call stub overflow"); 1999 return; 2000 } 2001 2002 int start = __ offset(); 2003 2004 __ relocate(static_stub_Relocation::spec(call_pc)); 2005 __ mov_metadata(rmethod, (Metadata*)NULL); 2006 __ movptr(rscratch1, 0); 2007 __ br(rscratch1); 2008 2009 assert(__ offset() - start <= call_stub_size(), "stub too big"); 2010 __ end_a_stub(); 2011 } 2012 2013 2014 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 2015 assert(exceptionOop->as_register() == r0, "must match"); 2016 assert(exceptionPC->as_register() == r3, "must match"); 2017 2018 // exception object is not added to oop map by LinearScan 2019 // (LinearScan assumes that no oops are in fixed registers) 2020 info->add_register_oop(exceptionOop); 2021 Runtime1::StubID unwind_id; 2022 2023 // get current pc information 2024 // pc is only needed if the method has an exception handler, the unwind code does not need it. 2025 int pc_for_athrow_offset = __ offset(); 2026 InternalAddress pc_for_athrow(__ pc()); 2027 __ adr(exceptionPC->as_register(), pc_for_athrow); 2028 add_call_info(pc_for_athrow_offset, info); // for exception handler 2029 2030 __ verify_not_null_oop(r0); 2031 // search an exception handler (r0: exception oop, r3: throwing pc) 2032 if (compilation()->has_fpu_code()) { 2033 unwind_id = Runtime1::handle_exception_id; 2034 } else { 2035 unwind_id = Runtime1::handle_exception_nofpu_id; 2036 } 2037 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 2038 2039 // FIXME: enough room for two byte trap ???? 2040 __ nop(); 2041 } 2042 2043 2044 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 2045 assert(exceptionOop->as_register() == r0, "must match"); 2046 2047 __ b(_unwind_handler_entry); 2048 } 2049 2050 2051 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 2052 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2053 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2054 2055 switch (left->type()) { 2056 case T_INT: { 2057 switch (code) { 2058 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break; 2059 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break; 2060 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break; 2061 default: 2062 ShouldNotReachHere(); 2063 break; 2064 } 2065 break; 2066 case T_LONG: 2067 case T_ADDRESS: 2068 case T_OBJECT: 2069 switch (code) { 2070 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break; 2071 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break; 2072 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break; 2073 default: 2074 ShouldNotReachHere(); 2075 break; 2076 } 2077 break; 2078 default: 2079 ShouldNotReachHere(); 2080 break; 2081 } 2082 } 2083 } 2084 2085 2086 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 2087 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2088 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2089 2090 switch (left->type()) { 2091 case T_INT: { 2092 switch (code) { 2093 case lir_shl: __ lslw (dreg, lreg, count); break; 2094 case lir_shr: __ asrw (dreg, lreg, count); break; 2095 case lir_ushr: __ lsrw (dreg, lreg, count); break; 2096 default: 2097 ShouldNotReachHere(); 2098 break; 2099 } 2100 break; 2101 case T_LONG: 2102 case T_ADDRESS: 2103 case T_OBJECT: 2104 switch (code) { 2105 case lir_shl: __ lsl (dreg, lreg, count); break; 2106 case lir_shr: __ asr (dreg, lreg, count); break; 2107 case lir_ushr: __ lsr (dreg, lreg, count); break; 2108 default: 2109 ShouldNotReachHere(); 2110 break; 2111 } 2112 break; 2113 default: 2114 ShouldNotReachHere(); 2115 break; 2116 } 2117 } 2118 } 2119 2120 2121 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 2122 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2123 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2124 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2125 __ str (r, Address(sp, offset_from_rsp_in_bytes)); 2126 } 2127 2128 2129 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 2130 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2131 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2132 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2133 __ mov (rscratch1, c); 2134 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2135 } 2136 2137 2138 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) { 2139 ShouldNotReachHere(); 2140 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2141 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2142 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2143 __ lea(rscratch1, __ constant_oop_address(o)); 2144 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2145 } 2146 2147 2148 // This code replaces a call to arraycopy; no exception may 2149 // be thrown in this code, they must be thrown in the System.arraycopy 2150 // activation frame; we could save some checks if this would not be the case 2151 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 2152 ciArrayKlass* default_type = op->expected_type(); 2153 Register src = op->src()->as_register(); 2154 Register dst = op->dst()->as_register(); 2155 Register src_pos = op->src_pos()->as_register(); 2156 Register dst_pos = op->dst_pos()->as_register(); 2157 Register length = op->length()->as_register(); 2158 Register tmp = op->tmp()->as_register(); 2159 2160 CodeStub* stub = op->stub(); 2161 int flags = op->flags(); 2162 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL; 2163 if (basic_type == T_ARRAY) basic_type = T_OBJECT; 2164 2165 // if we don't know anything, just go through the generic arraycopy 2166 if (default_type == NULL // || basic_type == T_OBJECT 2167 ) { 2168 Label done; 2169 assert(src == r1 && src_pos == r2, "mismatch in calling convention"); 2170 2171 // Save the arguments in case the generic arraycopy fails and we 2172 // have to fall back to the JNI stub 2173 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2174 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2175 __ str(src, Address(sp, 4*BytesPerWord)); 2176 2177 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy); 2178 address copyfunc_addr = StubRoutines::generic_arraycopy(); 2179 2180 // The arguments are in java calling convention so we shift them 2181 // to C convention 2182 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4); 2183 __ mov(c_rarg0, j_rarg0); 2184 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4); 2185 __ mov(c_rarg1, j_rarg1); 2186 assert_different_registers(c_rarg2, j_rarg3, j_rarg4); 2187 __ mov(c_rarg2, j_rarg2); 2188 assert_different_registers(c_rarg3, j_rarg4); 2189 __ mov(c_rarg3, j_rarg3); 2190 __ mov(c_rarg4, j_rarg4); 2191 if (copyfunc_addr == NULL) { // Use C version if stub was not generated 2192 __ mov(rscratch1, RuntimeAddress(C_entry)); 2193 __ blrt(rscratch1, 5, 0, 1); 2194 } else { 2195 #ifndef PRODUCT 2196 if (PrintC1Statistics) { 2197 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); 2198 } 2199 #endif 2200 __ far_call(RuntimeAddress(copyfunc_addr)); 2201 } 2202 2203 __ cbz(r0, *stub->continuation()); 2204 2205 // Reload values from the stack so they are where the stub 2206 // expects them. 2207 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2208 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2209 __ ldr(src, Address(sp, 4*BytesPerWord)); 2210 2211 if (copyfunc_addr != NULL) { 2212 // r0 is -1^K where K == partial copied count 2213 __ eonw(rscratch1, r0, 0); 2214 // adjust length down and src/end pos up by partial copied count 2215 __ subw(length, length, rscratch1); 2216 __ addw(src_pos, src_pos, rscratch1); 2217 __ addw(dst_pos, dst_pos, rscratch1); 2218 } 2219 __ b(*stub->entry()); 2220 2221 __ bind(*stub->continuation()); 2222 return; 2223 } 2224 2225 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point"); 2226 2227 int elem_size = type2aelembytes(basic_type); 2228 int shift_amount; 2229 int scale = exact_log2(elem_size); 2230 2231 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); 2232 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); 2233 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); 2234 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); 2235 2236 // test for NULL 2237 if (flags & LIR_OpArrayCopy::src_null_check) { 2238 __ cbz(src, *stub->entry()); 2239 } 2240 if (flags & LIR_OpArrayCopy::dst_null_check) { 2241 __ cbz(dst, *stub->entry()); 2242 } 2243 2244 // If the compiler was not able to prove that exact type of the source or the destination 2245 // of the arraycopy is an array type, check at runtime if the source or the destination is 2246 // an instance type. 2247 if (flags & LIR_OpArrayCopy::type_check) { 2248 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) { 2249 __ load_klass(tmp, dst); 2250 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2251 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2252 __ br(Assembler::GE, *stub->entry()); 2253 } 2254 2255 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) { 2256 __ load_klass(tmp, src); 2257 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2258 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2259 __ br(Assembler::GE, *stub->entry()); 2260 } 2261 } 2262 2263 // check if negative 2264 if (flags & LIR_OpArrayCopy::src_pos_positive_check) { 2265 __ cmpw(src_pos, 0); 2266 __ br(Assembler::LT, *stub->entry()); 2267 } 2268 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) { 2269 __ cmpw(dst_pos, 0); 2270 __ br(Assembler::LT, *stub->entry()); 2271 } 2272 2273 if (flags & LIR_OpArrayCopy::length_positive_check) { 2274 __ cmpw(length, 0); 2275 __ br(Assembler::LT, *stub->entry()); 2276 } 2277 2278 if (flags & LIR_OpArrayCopy::src_range_check) { 2279 __ addw(tmp, src_pos, length); 2280 __ ldrw(rscratch1, src_length_addr); 2281 __ cmpw(tmp, rscratch1); 2282 __ br(Assembler::HI, *stub->entry()); 2283 } 2284 if (flags & LIR_OpArrayCopy::dst_range_check) { 2285 __ addw(tmp, dst_pos, length); 2286 __ ldrw(rscratch1, dst_length_addr); 2287 __ cmpw(tmp, rscratch1); 2288 __ br(Assembler::HI, *stub->entry()); 2289 } 2290 2291 if (flags & LIR_OpArrayCopy::type_check) { 2292 // We don't know the array types are compatible 2293 if (basic_type != T_OBJECT) { 2294 // Simple test for basic type arrays 2295 if (UseCompressedClassPointers) { 2296 __ ldrw(tmp, src_klass_addr); 2297 __ ldrw(rscratch1, dst_klass_addr); 2298 __ cmpw(tmp, rscratch1); 2299 } else { 2300 __ ldr(tmp, src_klass_addr); 2301 __ ldr(rscratch1, dst_klass_addr); 2302 __ cmp(tmp, rscratch1); 2303 } 2304 __ br(Assembler::NE, *stub->entry()); 2305 } else { 2306 // For object arrays, if src is a sub class of dst then we can 2307 // safely do the copy. 2308 Label cont, slow; 2309 2310 #define PUSH(r1, r2) \ 2311 stp(r1, r2, __ pre(sp, -2 * wordSize)); 2312 2313 #define POP(r1, r2) \ 2314 ldp(r1, r2, __ post(sp, 2 * wordSize)); 2315 2316 __ PUSH(src, dst); 2317 2318 __ load_klass(src, src); 2319 __ load_klass(dst, dst); 2320 2321 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL); 2322 2323 __ PUSH(src, dst); 2324 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 2325 __ POP(src, dst); 2326 2327 __ cbnz(src, cont); 2328 2329 __ bind(slow); 2330 __ POP(src, dst); 2331 2332 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 2333 if (copyfunc_addr != NULL) { // use stub if available 2334 // src is not a sub class of dst so we have to do a 2335 // per-element check. 2336 2337 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 2338 if ((flags & mask) != mask) { 2339 // Check that at least both of them object arrays. 2340 assert(flags & mask, "one of the two should be known to be an object array"); 2341 2342 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 2343 __ load_klass(tmp, src); 2344 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 2345 __ load_klass(tmp, dst); 2346 } 2347 int lh_offset = in_bytes(Klass::layout_helper_offset()); 2348 Address klass_lh_addr(tmp, lh_offset); 2349 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 2350 __ ldrw(rscratch1, klass_lh_addr); 2351 __ mov(rscratch2, objArray_lh); 2352 __ eorw(rscratch1, rscratch1, rscratch2); 2353 __ cbnzw(rscratch1, *stub->entry()); 2354 } 2355 2356 // Spill because stubs can use any register they like and it's 2357 // easier to restore just those that we care about. 2358 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2359 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2360 __ str(src, Address(sp, 4*BytesPerWord)); 2361 2362 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2363 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2364 assert_different_registers(c_rarg0, dst, dst_pos, length); 2365 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2366 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2367 assert_different_registers(c_rarg1, dst, length); 2368 __ uxtw(c_rarg2, length); 2369 assert_different_registers(c_rarg2, dst); 2370 2371 __ load_klass(c_rarg4, dst); 2372 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset())); 2373 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset())); 2374 __ far_call(RuntimeAddress(copyfunc_addr)); 2375 2376 #ifndef PRODUCT 2377 if (PrintC1Statistics) { 2378 Label failed; 2379 __ cbnz(r0, failed); 2380 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt)); 2381 __ bind(failed); 2382 } 2383 #endif 2384 2385 __ cbz(r0, *stub->continuation()); 2386 2387 #ifndef PRODUCT 2388 if (PrintC1Statistics) { 2389 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt)); 2390 } 2391 #endif 2392 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1); 2393 2394 // Restore previously spilled arguments 2395 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2396 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2397 __ ldr(src, Address(sp, 4*BytesPerWord)); 2398 2399 // return value is -1^K where K is partial copied count 2400 __ eonw(rscratch1, r0, zr); 2401 // adjust length down and src/end pos up by partial copied count 2402 __ subw(length, length, rscratch1); 2403 __ addw(src_pos, src_pos, rscratch1); 2404 __ addw(dst_pos, dst_pos, rscratch1); 2405 } 2406 2407 __ b(*stub->entry()); 2408 2409 __ bind(cont); 2410 __ POP(src, dst); 2411 } 2412 } 2413 2414 #ifdef ASSERT 2415 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { 2416 // Sanity check the known type with the incoming class. For the 2417 // primitive case the types must match exactly with src.klass and 2418 // dst.klass each exactly matching the default type. For the 2419 // object array case, if no type check is needed then either the 2420 // dst type is exactly the expected type and the src type is a 2421 // subtype which we can't check or src is the same array as dst 2422 // but not necessarily exactly of type default_type. 2423 Label known_ok, halt; 2424 __ mov_metadata(tmp, default_type->constant_encoding()); 2425 if (UseCompressedClassPointers) { 2426 __ encode_klass_not_null(tmp); 2427 } 2428 2429 if (basic_type != T_OBJECT) { 2430 2431 if (UseCompressedClassPointers) { 2432 __ ldrw(rscratch1, dst_klass_addr); 2433 __ cmpw(tmp, rscratch1); 2434 } else { 2435 __ ldr(rscratch1, dst_klass_addr); 2436 __ cmp(tmp, rscratch1); 2437 } 2438 __ br(Assembler::NE, halt); 2439 if (UseCompressedClassPointers) { 2440 __ ldrw(rscratch1, src_klass_addr); 2441 __ cmpw(tmp, rscratch1); 2442 } else { 2443 __ ldr(rscratch1, src_klass_addr); 2444 __ cmp(tmp, rscratch1); 2445 } 2446 __ br(Assembler::EQ, known_ok); 2447 } else { 2448 if (UseCompressedClassPointers) { 2449 __ ldrw(rscratch1, dst_klass_addr); 2450 __ cmpw(tmp, rscratch1); 2451 } else { 2452 __ ldr(rscratch1, dst_klass_addr); 2453 __ cmp(tmp, rscratch1); 2454 } 2455 __ br(Assembler::EQ, known_ok); 2456 __ cmp(src, dst); 2457 __ br(Assembler::EQ, known_ok); 2458 } 2459 __ bind(halt); 2460 __ stop("incorrect type information in arraycopy"); 2461 __ bind(known_ok); 2462 } 2463 #endif 2464 2465 #ifndef PRODUCT 2466 if (PrintC1Statistics) { 2467 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type))); 2468 } 2469 #endif 2470 2471 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2472 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2473 assert_different_registers(c_rarg0, dst, dst_pos, length); 2474 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2475 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2476 assert_different_registers(c_rarg1, dst, length); 2477 __ uxtw(c_rarg2, length); 2478 assert_different_registers(c_rarg2, dst); 2479 2480 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 2481 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 2482 const char *name; 2483 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 2484 2485 CodeBlob *cb = CodeCache::find_blob(entry); 2486 if (cb) { 2487 __ far_call(RuntimeAddress(entry)); 2488 } else { 2489 __ call_VM_leaf(entry, 3); 2490 } 2491 2492 __ bind(*stub->continuation()); 2493 } 2494 2495 2496 2497 2498 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 2499 Register obj = op->obj_opr()->as_register(); // may not be an oop 2500 Register hdr = op->hdr_opr()->as_register(); 2501 Register lock = op->lock_opr()->as_register(); 2502 if (!UseFastLocking) { 2503 __ b(*op->stub()->entry()); 2504 } else if (op->code() == lir_lock) { 2505 Register scratch = noreg; 2506 if (UseBiasedLocking) { 2507 scratch = op->scratch_opr()->as_register(); 2508 } 2509 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2510 // add debug info for NullPointerException only if one is possible 2511 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry()); 2512 if (op->info() != NULL) { 2513 add_debug_info_for_null_check(null_check_offset, op->info()); 2514 } 2515 // done 2516 } else if (op->code() == lir_unlock) { 2517 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2518 __ unlock_object(hdr, obj, lock, *op->stub()->entry()); 2519 } else { 2520 Unimplemented(); 2521 } 2522 __ bind(*op->stub()->continuation()); 2523 } 2524 2525 2526 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 2527 ciMethod* method = op->profiled_method(); 2528 int bci = op->profiled_bci(); 2529 ciMethod* callee = op->profiled_callee(); 2530 2531 // Update counter for all call types 2532 ciMethodData* md = method->method_data_or_null(); 2533 assert(md != NULL, "Sanity"); 2534 ciProfileData* data = md->bci_to_data(bci); 2535 assert(data != NULL && data->is_CounterData(), "need CounterData for calls"); 2536 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 2537 Register mdo = op->mdo()->as_register(); 2538 __ mov_metadata(mdo, md->constant_encoding()); 2539 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 2540 // Perform additional virtual call profiling for invokevirtual and 2541 // invokeinterface bytecodes 2542 if (op->should_profile_receiver_type()) { 2543 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 2544 Register recv = op->recv()->as_register(); 2545 assert_different_registers(mdo, recv); 2546 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 2547 ciKlass* known_klass = op->known_holder(); 2548 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) { 2549 // We know the type that will be seen at this call site; we can 2550 // statically update the MethodData* rather than needing to do 2551 // dynamic tests on the receiver type 2552 2553 // NOTE: we should probably put a lock around this search to 2554 // avoid collisions by concurrent compilations 2555 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 2556 uint i; 2557 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2558 ciKlass* receiver = vc_data->receiver(i); 2559 if (known_klass->equals(receiver)) { 2560 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2561 __ addptr(data_addr, DataLayout::counter_increment); 2562 return; 2563 } 2564 } 2565 2566 // Receiver type not found in profile data; select an empty slot 2567 2568 // Note that this is less efficient than it should be because it 2569 // always does a write to the receiver part of the 2570 // VirtualCallData rather than just the first time 2571 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2572 ciKlass* receiver = vc_data->receiver(i); 2573 if (receiver == NULL) { 2574 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); 2575 __ mov_metadata(rscratch1, known_klass->constant_encoding()); 2576 __ lea(rscratch2, recv_addr); 2577 __ str(rscratch1, Address(rscratch2)); 2578 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2579 __ addptr(data_addr, DataLayout::counter_increment); 2580 return; 2581 } 2582 } 2583 } else { 2584 __ load_klass(recv, recv); 2585 Label update_done; 2586 type_profile_helper(mdo, md, data, recv, &update_done); 2587 // Receiver did not match any saved receiver and there is no empty row for it. 2588 // Increment total counter to indicate polymorphic case. 2589 __ addptr(counter_addr, DataLayout::counter_increment); 2590 2591 __ bind(update_done); 2592 } 2593 } else { 2594 // Static call 2595 __ addptr(counter_addr, DataLayout::counter_increment); 2596 } 2597 } 2598 2599 2600 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 2601 Unimplemented(); 2602 } 2603 2604 2605 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 2606 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 2607 } 2608 2609 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 2610 assert(op->crc()->is_single_cpu(), "crc must be register"); 2611 assert(op->val()->is_single_cpu(), "byte value must be register"); 2612 assert(op->result_opr()->is_single_cpu(), "result must be register"); 2613 Register crc = op->crc()->as_register(); 2614 Register val = op->val()->as_register(); 2615 Register res = op->result_opr()->as_register(); 2616 2617 assert_different_registers(val, crc, res); 2618 unsigned long offset; 2619 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset); 2620 if (offset) __ add(res, res, offset); 2621 2622 __ mvnw(crc, crc); // ~crc 2623 __ update_byte_crc32(crc, val, res); 2624 __ mvnw(res, crc); // ~crc 2625 } 2626 2627 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 2628 COMMENT("emit_profile_type {"); 2629 Register obj = op->obj()->as_register(); 2630 Register tmp = op->tmp()->as_pointer_register(); 2631 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 2632 ciKlass* exact_klass = op->exact_klass(); 2633 intptr_t current_klass = op->current_klass(); 2634 bool not_null = op->not_null(); 2635 bool no_conflict = op->no_conflict(); 2636 2637 Label update, next, none; 2638 2639 bool do_null = !not_null; 2640 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 2641 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 2642 2643 assert(do_null || do_update, "why are we here?"); 2644 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 2645 assert(mdo_addr.base() != rscratch1, "wrong register"); 2646 2647 __ verify_oop(obj); 2648 2649 if (tmp != obj) { 2650 __ mov(tmp, obj); 2651 } 2652 if (do_null) { 2653 __ cbnz(tmp, update); 2654 if (!TypeEntries::was_null_seen(current_klass)) { 2655 __ ldr(rscratch2, mdo_addr); 2656 __ orr(rscratch2, rscratch2, TypeEntries::null_seen); 2657 __ str(rscratch2, mdo_addr); 2658 } 2659 if (do_update) { 2660 #ifndef ASSERT 2661 __ b(next); 2662 } 2663 #else 2664 __ b(next); 2665 } 2666 } else { 2667 __ cbnz(tmp, update); 2668 __ stop("unexpected null obj"); 2669 #endif 2670 } 2671 2672 __ bind(update); 2673 2674 if (do_update) { 2675 #ifdef ASSERT 2676 if (exact_klass != NULL) { 2677 Label ok; 2678 __ load_klass(tmp, tmp); 2679 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2680 __ eor(rscratch1, tmp, rscratch1); 2681 __ cbz(rscratch1, ok); 2682 __ stop("exact klass and actual klass differ"); 2683 __ bind(ok); 2684 } 2685 #endif 2686 if (!no_conflict) { 2687 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) { 2688 if (exact_klass != NULL) { 2689 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2690 } else { 2691 __ load_klass(tmp, tmp); 2692 } 2693 2694 __ ldr(rscratch2, mdo_addr); 2695 __ eor(tmp, tmp, rscratch2); 2696 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2697 // klass seen before, nothing to do. The unknown bit may have been 2698 // set already but no need to check. 2699 __ cbz(rscratch1, next); 2700 2701 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2702 2703 if (TypeEntries::is_type_none(current_klass)) { 2704 __ cbz(rscratch2, none); 2705 __ cmp(rscratch2, TypeEntries::null_seen); 2706 __ br(Assembler::EQ, none); 2707 // There is a chance that the checks above (re-reading profiling 2708 // data from memory) fail if another thread has just set the 2709 // profiling to this obj's klass 2710 __ dmb(Assembler::ISHLD); 2711 __ ldr(rscratch2, mdo_addr); 2712 __ eor(tmp, tmp, rscratch2); 2713 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2714 __ cbz(rscratch1, next); 2715 } 2716 } else { 2717 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 2718 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 2719 2720 __ ldr(tmp, mdo_addr); 2721 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2722 } 2723 2724 // different than before. Cannot keep accurate profile. 2725 __ ldr(rscratch2, mdo_addr); 2726 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown); 2727 __ str(rscratch2, mdo_addr); 2728 2729 if (TypeEntries::is_type_none(current_klass)) { 2730 __ b(next); 2731 2732 __ bind(none); 2733 // first time here. Set profile type. 2734 __ str(tmp, mdo_addr); 2735 } 2736 } else { 2737 // There's a single possible klass at this profile point 2738 assert(exact_klass != NULL, "should be"); 2739 if (TypeEntries::is_type_none(current_klass)) { 2740 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2741 __ ldr(rscratch2, mdo_addr); 2742 __ eor(tmp, tmp, rscratch2); 2743 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2744 __ cbz(rscratch1, next); 2745 #ifdef ASSERT 2746 { 2747 Label ok; 2748 __ ldr(rscratch1, mdo_addr); 2749 __ cbz(rscratch1, ok); 2750 __ cmp(rscratch1, TypeEntries::null_seen); 2751 __ br(Assembler::EQ, ok); 2752 // may have been set by another thread 2753 __ dmb(Assembler::ISHLD); 2754 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2755 __ ldr(rscratch2, mdo_addr); 2756 __ eor(rscratch2, rscratch1, rscratch2); 2757 __ andr(rscratch2, rscratch2, TypeEntries::type_mask); 2758 __ cbz(rscratch2, ok); 2759 2760 __ stop("unexpected profiling mismatch"); 2761 __ bind(ok); 2762 } 2763 #endif 2764 // first time here. Set profile type. 2765 __ ldr(tmp, mdo_addr); 2766 } else { 2767 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 2768 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 2769 2770 __ ldr(tmp, mdo_addr); 2771 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2772 2773 __ orr(tmp, tmp, TypeEntries::type_unknown); 2774 __ str(tmp, mdo_addr); 2775 // FIXME: Write barrier needed here? 2776 } 2777 } 2778 2779 __ bind(next); 2780 } 2781 COMMENT("} emit_profile_type"); 2782 } 2783 2784 2785 void LIR_Assembler::align_backward_branch_target() { 2786 } 2787 2788 2789 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) { 2790 if (left->is_single_cpu()) { 2791 assert(dest->is_single_cpu(), "expect single result reg"); 2792 __ negw(dest->as_register(), left->as_register()); 2793 } else if (left->is_double_cpu()) { 2794 assert(dest->is_double_cpu(), "expect double result reg"); 2795 __ neg(dest->as_register_lo(), left->as_register_lo()); 2796 } else if (left->is_single_fpu()) { 2797 assert(dest->is_single_fpu(), "expect single float result reg"); 2798 __ fnegs(dest->as_float_reg(), left->as_float_reg()); 2799 } else { 2800 assert(left->is_double_fpu(), "expect double float operand reg"); 2801 assert(dest->is_double_fpu(), "expect double float result reg"); 2802 __ fnegd(dest->as_double_reg(), left->as_double_reg()); 2803 } 2804 } 2805 2806 2807 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) { 2808 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr())); 2809 } 2810 2811 2812 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 2813 assert(!tmp->is_valid(), "don't need temporary"); 2814 2815 CodeBlob *cb = CodeCache::find_blob(dest); 2816 if (cb) { 2817 __ far_call(RuntimeAddress(dest)); 2818 } else { 2819 __ mov(rscratch1, RuntimeAddress(dest)); 2820 int len = args->length(); 2821 int type = 0; 2822 if (! result->is_illegal()) { 2823 switch (result->type()) { 2824 case T_VOID: 2825 type = 0; 2826 break; 2827 case T_INT: 2828 case T_LONG: 2829 case T_OBJECT: 2830 type = 1; 2831 break; 2832 case T_FLOAT: 2833 type = 2; 2834 break; 2835 case T_DOUBLE: 2836 type = 3; 2837 break; 2838 default: 2839 ShouldNotReachHere(); 2840 break; 2841 } 2842 } 2843 int num_gpargs = 0; 2844 int num_fpargs = 0; 2845 for (int i = 0; i < args->length(); i++) { 2846 LIR_Opr arg = args->at(i); 2847 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) { 2848 num_fpargs++; 2849 } else { 2850 num_gpargs++; 2851 } 2852 } 2853 __ blrt(rscratch1, num_gpargs, num_fpargs, type); 2854 } 2855 2856 if (info != NULL) { 2857 add_call_info_here(info); 2858 } 2859 __ maybe_isb(); 2860 } 2861 2862 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 2863 if (dest->is_address() || src->is_address()) { 2864 move_op(src, dest, type, lir_patch_none, info, 2865 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false); 2866 } else { 2867 ShouldNotReachHere(); 2868 } 2869 } 2870 2871 #ifdef ASSERT 2872 // emit run-time assertion 2873 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 2874 assert(op->code() == lir_assert, "must be"); 2875 2876 if (op->in_opr1()->is_valid()) { 2877 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 2878 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 2879 } else { 2880 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 2881 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 2882 } 2883 2884 Label ok; 2885 if (op->condition() != lir_cond_always) { 2886 Assembler::Condition acond = Assembler::AL; 2887 switch (op->condition()) { 2888 case lir_cond_equal: acond = Assembler::EQ; break; 2889 case lir_cond_notEqual: acond = Assembler::NE; break; 2890 case lir_cond_less: acond = Assembler::LT; break; 2891 case lir_cond_lessEqual: acond = Assembler::LE; break; 2892 case lir_cond_greaterEqual: acond = Assembler::GE; break; 2893 case lir_cond_greater: acond = Assembler::GT; break; 2894 case lir_cond_belowEqual: acond = Assembler::LS; break; 2895 case lir_cond_aboveEqual: acond = Assembler::HS; break; 2896 default: ShouldNotReachHere(); 2897 } 2898 __ br(acond, ok); 2899 } 2900 if (op->halt()) { 2901 const char* str = __ code_string(op->msg()); 2902 __ stop(str); 2903 } else { 2904 breakpoint(); 2905 } 2906 __ bind(ok); 2907 } 2908 #endif 2909 2910 #ifndef PRODUCT 2911 #define COMMENT(x) do { __ block_comment(x); } while (0) 2912 #else 2913 #define COMMENT(x) 2914 #endif 2915 2916 void LIR_Assembler::membar() { 2917 COMMENT("membar"); 2918 __ membar(MacroAssembler::AnyAny); 2919 } 2920 2921 void LIR_Assembler::membar_acquire() { 2922 __ membar(Assembler::LoadLoad|Assembler::LoadStore); 2923 } 2924 2925 void LIR_Assembler::membar_release() { 2926 __ membar(Assembler::LoadStore|Assembler::StoreStore); 2927 } 2928 2929 void LIR_Assembler::membar_loadload() { 2930 __ membar(Assembler::LoadLoad); 2931 } 2932 2933 void LIR_Assembler::membar_storestore() { 2934 __ membar(MacroAssembler::StoreStore); 2935 } 2936 2937 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); } 2938 2939 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); } 2940 2941 void LIR_Assembler::on_spin_wait() { 2942 Unimplemented(); 2943 } 2944 2945 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 2946 __ mov(result_reg->as_register(), rthread); 2947 } 2948 2949 2950 void LIR_Assembler::peephole(LIR_List *lir) { 2951 #if 0 2952 if (tableswitch_count >= max_tableswitches) 2953 return; 2954 2955 /* 2956 This finite-state automaton recognizes sequences of compare-and- 2957 branch instructions. We will turn them into a tableswitch. You 2958 could argue that C1 really shouldn't be doing this sort of 2959 optimization, but without it the code is really horrible. 2960 */ 2961 2962 enum { start_s, cmp1_s, beq_s, cmp_s } state; 2963 int first_key, last_key = -2147483648; 2964 int next_key = 0; 2965 int start_insn = -1; 2966 int last_insn = -1; 2967 Register reg = noreg; 2968 LIR_Opr reg_opr; 2969 state = start_s; 2970 2971 LIR_OpList* inst = lir->instructions_list(); 2972 for (int i = 0; i < inst->length(); i++) { 2973 LIR_Op* op = inst->at(i); 2974 switch (state) { 2975 case start_s: 2976 first_key = -1; 2977 start_insn = i; 2978 switch (op->code()) { 2979 case lir_cmp: 2980 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 2981 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 2982 if (opr1->is_cpu_register() && opr1->is_single_cpu() 2983 && opr2->is_constant() 2984 && opr2->type() == T_INT) { 2985 reg_opr = opr1; 2986 reg = opr1->as_register(); 2987 first_key = opr2->as_constant_ptr()->as_jint(); 2988 next_key = first_key + 1; 2989 state = cmp_s; 2990 goto next_state; 2991 } 2992 break; 2993 } 2994 break; 2995 case cmp_s: 2996 switch (op->code()) { 2997 case lir_branch: 2998 if (op->as_OpBranch()->cond() == lir_cond_equal) { 2999 state = beq_s; 3000 last_insn = i; 3001 goto next_state; 3002 } 3003 } 3004 state = start_s; 3005 break; 3006 case beq_s: 3007 switch (op->code()) { 3008 case lir_cmp: { 3009 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3010 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3011 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3012 && opr1->as_register() == reg 3013 && opr2->is_constant() 3014 && opr2->type() == T_INT 3015 && opr2->as_constant_ptr()->as_jint() == next_key) { 3016 last_key = next_key; 3017 next_key++; 3018 state = cmp_s; 3019 goto next_state; 3020 } 3021 } 3022 } 3023 last_key = next_key; 3024 state = start_s; 3025 break; 3026 default: 3027 assert(false, "impossible state"); 3028 } 3029 if (state == start_s) { 3030 if (first_key < last_key - 5L && reg != noreg) { 3031 { 3032 // printf("found run register %d starting at insn %d low value %d high value %d\n", 3033 // reg->encoding(), 3034 // start_insn, first_key, last_key); 3035 // for (int i = 0; i < inst->length(); i++) { 3036 // inst->at(i)->print(); 3037 // tty->print("\n"); 3038 // } 3039 // tty->print("\n"); 3040 } 3041 3042 struct tableswitch *sw = &switches[tableswitch_count]; 3043 sw->_insn_index = start_insn, sw->_first_key = first_key, 3044 sw->_last_key = last_key, sw->_reg = reg; 3045 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after)); 3046 { 3047 // Insert the new table of branches 3048 int offset = last_insn; 3049 for (int n = first_key; n < last_key; n++) { 3050 inst->insert_before 3051 (last_insn + 1, 3052 new LIR_OpBranch(lir_cond_always, T_ILLEGAL, 3053 inst->at(offset)->as_OpBranch()->label())); 3054 offset -= 2, i++; 3055 } 3056 } 3057 // Delete all the old compare-and-branch instructions 3058 for (int n = first_key; n < last_key; n++) { 3059 inst->remove_at(start_insn); 3060 inst->remove_at(start_insn); 3061 } 3062 // Insert the tableswitch instruction 3063 inst->insert_before(start_insn, 3064 new LIR_Op2(lir_cmp, lir_cond_always, 3065 LIR_OprFact::intConst(tableswitch_count), 3066 reg_opr)); 3067 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches)); 3068 tableswitch_count++; 3069 } 3070 reg = noreg; 3071 last_key = -2147483648; 3072 } 3073 next_state: 3074 ; 3075 } 3076 #endif 3077 } 3078 3079 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { 3080 Address addr = as_Address(src->as_address_ptr()); 3081 BasicType type = src->type(); 3082 bool is_oop = type == T_OBJECT || type == T_ARRAY; 3083 3084 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr); 3085 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr); 3086 3087 switch(type) { 3088 case T_INT: 3089 xchg = &MacroAssembler::atomic_xchgalw; 3090 add = &MacroAssembler::atomic_addalw; 3091 break; 3092 case T_LONG: 3093 xchg = &MacroAssembler::atomic_xchgal; 3094 add = &MacroAssembler::atomic_addal; 3095 break; 3096 case T_OBJECT: 3097 case T_ARRAY: 3098 if (UseCompressedOops) { 3099 xchg = &MacroAssembler::atomic_xchgalw; 3100 add = &MacroAssembler::atomic_addalw; 3101 } else { 3102 xchg = &MacroAssembler::atomic_xchgal; 3103 add = &MacroAssembler::atomic_addal; 3104 } 3105 break; 3106 default: 3107 ShouldNotReachHere(); 3108 xchg = &MacroAssembler::atomic_xchgal; 3109 add = &MacroAssembler::atomic_addal; // unreachable 3110 } 3111 3112 switch (code) { 3113 case lir_xadd: 3114 { 3115 RegisterOrConstant inc; 3116 Register tmp = as_reg(tmp_op); 3117 Register dst = as_reg(dest); 3118 if (data->is_constant()) { 3119 inc = RegisterOrConstant(as_long(data)); 3120 assert_different_registers(dst, addr.base(), tmp, 3121 rscratch1, rscratch2); 3122 } else { 3123 inc = RegisterOrConstant(as_reg(data)); 3124 assert_different_registers(inc.as_register(), dst, addr.base(), tmp, 3125 rscratch1, rscratch2); 3126 } 3127 __ lea(tmp, addr); 3128 (_masm->*add)(dst, inc, tmp); 3129 break; 3130 } 3131 case lir_xchg: 3132 { 3133 Register tmp = tmp_op->as_register(); 3134 Register obj = as_reg(data); 3135 Register dst = as_reg(dest); 3136 if (is_oop && UseCompressedOops) { 3137 __ encode_heap_oop(rscratch2, obj); 3138 obj = rscratch2; 3139 } 3140 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst); 3141 __ lea(tmp, addr); 3142 (_masm->*xchg)(dst, obj, tmp); 3143 if (is_oop && UseCompressedOops) { 3144 __ decode_heap_oop(dst); 3145 } 3146 } 3147 break; 3148 default: 3149 ShouldNotReachHere(); 3150 } 3151 __ membar(__ AnyAny); 3152 } 3153 3154 #undef __