1 /* 2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, Red Hat Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "asm/assembler.hpp" 28 #include "c1/c1_CodeStubs.hpp" 29 #include "c1/c1_Compilation.hpp" 30 #include "c1/c1_LIRAssembler.hpp" 31 #include "c1/c1_MacroAssembler.hpp" 32 #include "c1/c1_Runtime1.hpp" 33 #include "c1/c1_ValueStack.hpp" 34 #include "ci/ciArrayKlass.hpp" 35 #include "ci/ciInstance.hpp" 36 #include "gc/shared/barrierSet.hpp" 37 #include "gc/shared/cardTableModRefBS.hpp" 38 #include "gc/shared/collectedHeap.hpp" 39 #include "nativeInst_aarch64.hpp" 40 #include "oops/objArrayKlass.hpp" 41 #include "runtime/sharedRuntime.hpp" 42 #include "vmreg_aarch64.inline.hpp" 43 44 45 46 #ifndef PRODUCT 47 #define COMMENT(x) do { __ block_comment(x); } while (0) 48 #else 49 #define COMMENT(x) 50 #endif 51 52 NEEDS_CLEANUP // remove this definitions ? 53 const Register IC_Klass = rscratch2; // where the IC klass is cached 54 const Register SYNC_header = r0; // synchronization header 55 const Register SHIFT_count = r0; // where count for shift operations must be 56 57 #define __ _masm-> 58 59 60 static void select_different_registers(Register preserve, 61 Register extra, 62 Register &tmp1, 63 Register &tmp2) { 64 if (tmp1 == preserve) { 65 assert_different_registers(tmp1, tmp2, extra); 66 tmp1 = extra; 67 } else if (tmp2 == preserve) { 68 assert_different_registers(tmp1, tmp2, extra); 69 tmp2 = extra; 70 } 71 assert_different_registers(preserve, tmp1, tmp2); 72 } 73 74 75 76 static void select_different_registers(Register preserve, 77 Register extra, 78 Register &tmp1, 79 Register &tmp2, 80 Register &tmp3) { 81 if (tmp1 == preserve) { 82 assert_different_registers(tmp1, tmp2, tmp3, extra); 83 tmp1 = extra; 84 } else if (tmp2 == preserve) { 85 assert_different_registers(tmp1, tmp2, tmp3, extra); 86 tmp2 = extra; 87 } else if (tmp3 == preserve) { 88 assert_different_registers(tmp1, tmp2, tmp3, extra); 89 tmp3 = extra; 90 } 91 assert_different_registers(preserve, tmp1, tmp2, tmp3); 92 } 93 94 95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } 96 97 98 LIR_Opr LIR_Assembler::receiverOpr() { 99 return FrameMap::receiver_opr; 100 } 101 102 LIR_Opr LIR_Assembler::osrBufferPointer() { 103 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 104 } 105 106 //--------------fpu register translations----------------------- 107 108 109 address LIR_Assembler::float_constant(float f) { 110 address const_addr = __ float_constant(f); 111 if (const_addr == NULL) { 112 bailout("const section overflow"); 113 return __ code()->consts()->start(); 114 } else { 115 return const_addr; 116 } 117 } 118 119 120 address LIR_Assembler::double_constant(double d) { 121 address const_addr = __ double_constant(d); 122 if (const_addr == NULL) { 123 bailout("const section overflow"); 124 return __ code()->consts()->start(); 125 } else { 126 return const_addr; 127 } 128 } 129 130 address LIR_Assembler::int_constant(jlong n) { 131 address const_addr = __ long_constant(n); 132 if (const_addr == NULL) { 133 bailout("const section overflow"); 134 return __ code()->consts()->start(); 135 } else { 136 return const_addr; 137 } 138 } 139 140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); } 141 142 void LIR_Assembler::reset_FPU() { Unimplemented(); } 143 144 void LIR_Assembler::fpop() { Unimplemented(); } 145 146 void LIR_Assembler::fxch(int i) { Unimplemented(); } 147 148 void LIR_Assembler::fld(int i) { Unimplemented(); } 149 150 void LIR_Assembler::ffree(int i) { Unimplemented(); } 151 152 void LIR_Assembler::breakpoint() { Unimplemented(); } 153 154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); } 155 156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); } 157 158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; } 159 //------------------------------------------- 160 161 static Register as_reg(LIR_Opr op) { 162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register(); 163 } 164 165 static jlong as_long(LIR_Opr data) { 166 jlong result; 167 switch (data->type()) { 168 case T_INT: 169 result = (data->as_jint()); 170 break; 171 case T_LONG: 172 result = (data->as_jlong()); 173 break; 174 default: 175 ShouldNotReachHere(); 176 result = 0; // unreachable 177 } 178 return result; 179 } 180 181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 182 Register base = addr->base()->as_pointer_register(); 183 LIR_Opr opr = addr->index(); 184 if (opr->is_cpu_register()) { 185 Register index; 186 if (opr->is_single_cpu()) 187 index = opr->as_register(); 188 else 189 index = opr->as_register_lo(); 190 assert(addr->disp() == 0, "must be"); 191 switch(opr->type()) { 192 case T_INT: 193 return Address(base, index, Address::sxtw(addr->scale())); 194 case T_LONG: 195 return Address(base, index, Address::lsl(addr->scale())); 196 default: 197 ShouldNotReachHere(); 198 } 199 } else { 200 intptr_t addr_offset = intptr_t(addr->disp()); 201 if (Address::offset_ok_for_immed(addr_offset, addr->scale())) 202 return Address(base, addr_offset, Address::lsl(addr->scale())); 203 else { 204 __ mov(tmp, addr_offset); 205 return Address(base, tmp, Address::lsl(addr->scale())); 206 } 207 } 208 return Address(); 209 } 210 211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 212 ShouldNotReachHere(); 213 return Address(); 214 } 215 216 Address LIR_Assembler::as_Address(LIR_Address* addr) { 217 return as_Address(addr, rscratch1); 218 } 219 220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 221 return as_Address(addr, rscratch1); // Ouch 222 // FIXME: This needs to be much more clever. See x86. 223 } 224 225 226 void LIR_Assembler::osr_entry() { 227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 228 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 229 ValueStack* entry_state = osr_entry->state(); 230 int number_of_locks = entry_state->locks_size(); 231 232 // we jump here if osr happens with the interpreter 233 // state set up to continue at the beginning of the 234 // loop that triggered osr - in particular, we have 235 // the following registers setup: 236 // 237 // r2: osr buffer 238 // 239 240 // build frame 241 ciMethod* m = compilation()->method(); 242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 243 244 // OSR buffer is 245 // 246 // locals[nlocals-1..0] 247 // monitors[0..number_of_locks] 248 // 249 // locals is a direct copy of the interpreter frame so in the osr buffer 250 // so first slot in the local array is the last local from the interpreter 251 // and last slot is local[0] (receiver) from the interpreter 252 // 253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 255 // in the interpreter frame (the method lock if a sync method) 256 257 // Initialize monitors in the compiled activation. 258 // r2: pointer to osr buffer 259 // 260 // All other registers are dead at this point and the locals will be 261 // copied into place by code emitted in the IR. 262 263 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 265 int monitor_offset = BytesPerWord * method()->max_locals() + 266 (2 * BytesPerWord) * (number_of_locks - 1); 267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 268 // the OSR buffer using 2 word entries: first the lock and then 269 // the oop. 270 for (int i = 0; i < number_of_locks; i++) { 271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 272 #ifdef ASSERT 273 // verify the interpreter's monitor has a non-null object 274 { 275 Label L; 276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 277 __ cbnz(rscratch1, L); 278 __ stop("locked object is NULL"); 279 __ bind(L); 280 } 281 #endif 282 __ ldr(r19, Address(OSR_buf, slot_offset + 0)); 283 __ str(r19, frame_map()->address_for_monitor_lock(i)); 284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 285 __ str(r19, frame_map()->address_for_monitor_object(i)); 286 } 287 } 288 } 289 290 291 // inline cache check; done before the frame is built. 292 int LIR_Assembler::check_icache() { 293 Register receiver = FrameMap::receiver_opr->as_register(); 294 Register ic_klass = IC_Klass; 295 int start_offset = __ offset(); 296 __ inline_cache_check(receiver, ic_klass); 297 298 // if icache check fails, then jump to runtime routine 299 // Note: RECEIVER must still contain the receiver! 300 Label dont; 301 __ br(Assembler::EQ, dont); 302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 303 304 // We align the verified entry point unless the method body 305 // (including its inline cache check) will fit in a single 64-byte 306 // icache line. 307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) { 308 // force alignment after the cache check. 309 __ align(CodeEntryAlignment); 310 } 311 312 __ bind(dont); 313 return start_offset; 314 } 315 316 317 void LIR_Assembler::jobject2reg(jobject o, Register reg) { 318 if (o == NULL) { 319 __ mov(reg, zr); 320 } else { 321 __ movoop(reg, o, /*immediate*/true); 322 } 323 } 324 325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) { 326 address target = NULL; 327 relocInfo::relocType reloc_type = relocInfo::none; 328 329 switch (patching_id(info)) { 330 case PatchingStub::access_field_id: 331 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 332 reloc_type = relocInfo::section_word_type; 333 break; 334 case PatchingStub::load_klass_id: 335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 336 reloc_type = relocInfo::metadata_type; 337 break; 338 case PatchingStub::load_mirror_id: 339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 340 reloc_type = relocInfo::oop_type; 341 break; 342 case PatchingStub::load_appendix_id: 343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 344 reloc_type = relocInfo::oop_type; 345 break; 346 default: ShouldNotReachHere(); 347 } 348 349 __ far_call(RuntimeAddress(target)); 350 add_call_info_here(info); 351 } 352 353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) { 354 deoptimize_trap(info); 355 } 356 357 358 // This specifies the rsp decrement needed to build the frame 359 int LIR_Assembler::initial_frame_size_in_bytes() const { 360 // if rounding, must let FrameMap know! 361 362 // The frame_map records size in slots (32bit word) 363 364 // subtract two words to account for return address and link 365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size; 366 } 367 368 369 int LIR_Assembler::emit_exception_handler() { 370 // if the last instruction is a call (typically to do a throw which 371 // is coming at the end after block reordering) the return address 372 // must still point into the code area in order to avoid assertion 373 // failures when searching for the corresponding bci => add a nop 374 // (was bug 5/14/1999 - gri) 375 __ nop(); 376 377 // generate code for exception handler 378 address handler_base = __ start_a_stub(exception_handler_size()); 379 if (handler_base == NULL) { 380 // not enough space left for the handler 381 bailout("exception handler overflow"); 382 return -1; 383 } 384 385 int offset = code_offset(); 386 387 // the exception oop and pc are in r0, and r3 388 // no other registers need to be preserved, so invalidate them 389 __ invalidate_registers(false, true, true, false, true, true); 390 391 // check that there is really an exception 392 __ verify_not_null_oop(r0); 393 394 // search an exception handler (r0: exception oop, r3: throwing pc) 395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here(); 396 guarantee(code_offset() - offset <= exception_handler_size(), "overflow"); 397 __ end_a_stub(); 398 399 return offset; 400 } 401 402 403 // Emit the code to remove the frame from the stack in the exception 404 // unwind path. 405 int LIR_Assembler::emit_unwind_handler() { 406 #ifndef PRODUCT 407 if (CommentedAssembly) { 408 _masm->block_comment("Unwind handler"); 409 } 410 #endif 411 412 int offset = code_offset(); 413 414 // Fetch the exception from TLS and clear out exception related thread state 415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset())); 416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset())); 417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset())); 418 419 __ bind(_unwind_handler_entry); 420 __ verify_not_null_oop(r0); 421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 422 __ mov(r19, r0); // Preserve the exception 423 } 424 425 // Preform needed unlocking 426 MonitorExitStub* stub = NULL; 427 if (method()->is_synchronized()) { 428 monitor_address(0, FrameMap::r0_opr); 429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0); 430 __ unlock_object(r5, r4, r0, *stub->entry()); 431 __ bind(*stub->continuation()); 432 } 433 434 if (compilation()->env()->dtrace_method_probes()) { 435 __ call_Unimplemented(); 436 #if 0 437 __ movptr(Address(rsp, 0), rax); 438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding()); 439 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit))); 440 #endif 441 } 442 443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 444 __ mov(r0, r19); // Restore the exception 445 } 446 447 // remove the activation and dispatch to the unwind handler 448 __ block_comment("remove_frame and dispatch to the unwind handler"); 449 __ remove_frame(initial_frame_size_in_bytes()); 450 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id))); 451 452 // Emit the slow path assembly 453 if (stub != NULL) { 454 stub->emit_code(this); 455 } 456 457 return offset; 458 } 459 460 461 int LIR_Assembler::emit_deopt_handler() { 462 // if the last instruction is a call (typically to do a throw which 463 // is coming at the end after block reordering) the return address 464 // must still point into the code area in order to avoid assertion 465 // failures when searching for the corresponding bci => add a nop 466 // (was bug 5/14/1999 - gri) 467 __ nop(); 468 469 // generate code for exception handler 470 address handler_base = __ start_a_stub(deopt_handler_size()); 471 if (handler_base == NULL) { 472 // not enough space left for the handler 473 bailout("deopt handler overflow"); 474 return -1; 475 } 476 477 int offset = code_offset(); 478 479 __ adr(lr, pc()); 480 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 481 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); 482 __ end_a_stub(); 483 484 return offset; 485 } 486 487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) { 488 _masm->code_section()->relocate(adr, relocInfo::poll_type); 489 int pc_offset = code_offset(); 490 flush_debug_info(pc_offset); 491 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset); 492 if (info->exception_handlers() != NULL) { 493 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers()); 494 } 495 } 496 497 // Rather than take a segfault when the polling page is protected, 498 // explicitly check for a safepoint in progress and if there is one, 499 // fake a call to the handler as if a segfault had been caught. 500 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) { 501 __ mov(rscratch1, SafepointSynchronize::address_of_state()); 502 __ ldrb(rscratch1, Address(rscratch1)); 503 Label nope, poll; 504 __ cbz(rscratch1, nope); 505 __ block_comment("safepoint"); 506 __ enter(); 507 __ push(0x3, sp); // r0 & r1 508 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1 509 __ adr(r0, poll); 510 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset())); 511 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub)); 512 __ blrt(rscratch1, 1, 0, 1); 513 __ maybe_isb(); 514 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1 515 __ mov(rscratch1, r0); 516 __ pop(0x3, sp); // r0 & r1 517 __ leave(); 518 __ br(rscratch1); 519 address polling_page(os::get_polling_page()); 520 assert(os::is_poll_address(polling_page), "should be"); 521 unsigned long off; 522 __ adrp(rscratch1, Address(polling_page, rtype), off); 523 __ bind(poll); 524 if (info) 525 add_debug_info_for_branch(info); // This isn't just debug info: 526 // it's the oop map 527 else 528 __ code_section()->relocate(pc(), rtype); 529 __ ldrw(zr, Address(rscratch1, off)); 530 __ bind(nope); 531 } 532 533 void LIR_Assembler::return_op(LIR_Opr result) { 534 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,"); 535 536 // Pop the stack before the safepoint code 537 __ remove_frame(initial_frame_size_in_bytes()); 538 539 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) { 540 __ reserved_stack_check(); 541 } 542 543 address polling_page(os::get_polling_page()); 544 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type); 545 __ ret(lr); 546 } 547 548 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 549 address polling_page(os::get_polling_page()); 550 guarantee(info != NULL, "Shouldn't be NULL"); 551 assert(os::is_poll_address(polling_page), "should be"); 552 unsigned long off; 553 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off); 554 assert(off == 0, "must be"); 555 add_debug_info_for_branch(info); // This isn't just debug info: 556 // it's the oop map 557 __ read_polling_page(rscratch1, relocInfo::poll_type); 558 return __ offset(); 559 } 560 561 562 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 563 if (from_reg == r31_sp) 564 from_reg = sp; 565 if (to_reg == r31_sp) 566 to_reg = sp; 567 __ mov(to_reg, from_reg); 568 } 569 570 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); } 571 572 573 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 574 assert(src->is_constant(), "should not call otherwise"); 575 assert(dest->is_register(), "should not call otherwise"); 576 LIR_Const* c = src->as_constant_ptr(); 577 578 switch (c->type()) { 579 case T_INT: { 580 assert(patch_code == lir_patch_none, "no patching handled here"); 581 __ movw(dest->as_register(), c->as_jint()); 582 break; 583 } 584 585 case T_ADDRESS: { 586 assert(patch_code == lir_patch_none, "no patching handled here"); 587 __ mov(dest->as_register(), c->as_jint()); 588 break; 589 } 590 591 case T_LONG: { 592 assert(patch_code == lir_patch_none, "no patching handled here"); 593 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong()); 594 break; 595 } 596 597 case T_OBJECT: { 598 if (patch_code == lir_patch_none) { 599 jobject2reg(c->as_jobject(), dest->as_register()); 600 } else { 601 jobject2reg_with_patching(dest->as_register(), info); 602 } 603 break; 604 } 605 606 case T_METADATA: { 607 if (patch_code != lir_patch_none) { 608 klass2reg_with_patching(dest->as_register(), info); 609 } else { 610 __ mov_metadata(dest->as_register(), c->as_metadata()); 611 } 612 break; 613 } 614 615 case T_FLOAT: { 616 if (__ operand_valid_for_float_immediate(c->as_jfloat())) { 617 __ fmovs(dest->as_float_reg(), (c->as_jfloat())); 618 } else { 619 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat()))); 620 __ ldrs(dest->as_float_reg(), Address(rscratch1)); 621 } 622 break; 623 } 624 625 case T_DOUBLE: { 626 if (__ operand_valid_for_float_immediate(c->as_jdouble())) { 627 __ fmovd(dest->as_double_reg(), (c->as_jdouble())); 628 } else { 629 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble()))); 630 __ ldrd(dest->as_double_reg(), Address(rscratch1)); 631 } 632 break; 633 } 634 635 default: 636 ShouldNotReachHere(); 637 } 638 } 639 640 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 641 LIR_Const* c = src->as_constant_ptr(); 642 switch (c->type()) { 643 case T_OBJECT: 644 { 645 if (! c->as_jobject()) 646 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 647 else { 648 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL); 649 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 650 } 651 } 652 break; 653 case T_ADDRESS: 654 { 655 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL); 656 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 657 } 658 case T_INT: 659 case T_FLOAT: 660 { 661 Register reg = zr; 662 if (c->as_jint_bits() == 0) 663 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 664 else { 665 __ movw(rscratch1, c->as_jint_bits()); 666 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix())); 667 } 668 } 669 break; 670 case T_LONG: 671 case T_DOUBLE: 672 { 673 Register reg = zr; 674 if (c->as_jlong_bits() == 0) 675 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(), 676 lo_word_offset_in_bytes)); 677 else { 678 __ mov(rscratch1, (intptr_t)c->as_jlong_bits()); 679 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(), 680 lo_word_offset_in_bytes)); 681 } 682 } 683 break; 684 default: 685 ShouldNotReachHere(); 686 } 687 } 688 689 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 690 assert(src->is_constant(), "should not call otherwise"); 691 LIR_Const* c = src->as_constant_ptr(); 692 LIR_Address* to_addr = dest->as_address_ptr(); 693 694 void (Assembler::* insn)(Register Rt, const Address &adr); 695 696 switch (type) { 697 case T_ADDRESS: 698 assert(c->as_jint() == 0, "should be"); 699 insn = &Assembler::str; 700 break; 701 case T_LONG: 702 assert(c->as_jlong() == 0, "should be"); 703 insn = &Assembler::str; 704 break; 705 case T_INT: 706 assert(c->as_jint() == 0, "should be"); 707 insn = &Assembler::strw; 708 break; 709 case T_OBJECT: 710 case T_ARRAY: 711 assert(c->as_jobject() == 0, "should be"); 712 if (UseCompressedOops && !wide) { 713 insn = &Assembler::strw; 714 } else { 715 insn = &Assembler::str; 716 } 717 break; 718 case T_CHAR: 719 case T_SHORT: 720 assert(c->as_jint() == 0, "should be"); 721 insn = &Assembler::strh; 722 break; 723 case T_BOOLEAN: 724 case T_BYTE: 725 assert(c->as_jint() == 0, "should be"); 726 insn = &Assembler::strb; 727 break; 728 default: 729 ShouldNotReachHere(); 730 insn = &Assembler::str; // unreachable 731 } 732 733 if (info) add_debug_info_for_null_check_here(info); 734 (_masm->*insn)(zr, as_Address(to_addr, rscratch1)); 735 } 736 737 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 738 assert(src->is_register(), "should not call otherwise"); 739 assert(dest->is_register(), "should not call otherwise"); 740 741 // move between cpu-registers 742 if (dest->is_single_cpu()) { 743 if (src->type() == T_LONG) { 744 // Can do LONG -> OBJECT 745 move_regs(src->as_register_lo(), dest->as_register()); 746 return; 747 } 748 assert(src->is_single_cpu(), "must match"); 749 if (src->type() == T_OBJECT) { 750 __ verify_oop(src->as_register()); 751 } 752 move_regs(src->as_register(), dest->as_register()); 753 754 } else if (dest->is_double_cpu()) { 755 if (src->type() == T_OBJECT || src->type() == T_ARRAY) { 756 // Surprising to me but we can see move of a long to t_object 757 __ verify_oop(src->as_register()); 758 move_regs(src->as_register(), dest->as_register_lo()); 759 return; 760 } 761 assert(src->is_double_cpu(), "must match"); 762 Register f_lo = src->as_register_lo(); 763 Register f_hi = src->as_register_hi(); 764 Register t_lo = dest->as_register_lo(); 765 Register t_hi = dest->as_register_hi(); 766 assert(f_hi == f_lo, "must be same"); 767 assert(t_hi == t_lo, "must be same"); 768 move_regs(f_lo, t_lo); 769 770 } else if (dest->is_single_fpu()) { 771 __ fmovs(dest->as_float_reg(), src->as_float_reg()); 772 773 } else if (dest->is_double_fpu()) { 774 __ fmovd(dest->as_double_reg(), src->as_double_reg()); 775 776 } else { 777 ShouldNotReachHere(); 778 } 779 } 780 781 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 782 if (src->is_single_cpu()) { 783 if (type == T_ARRAY || type == T_OBJECT) { 784 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 785 __ verify_oop(src->as_register()); 786 } else if (type == T_METADATA || type == T_DOUBLE) { 787 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 788 } else { 789 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 790 } 791 792 } else if (src->is_double_cpu()) { 793 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes); 794 __ str(src->as_register_lo(), dest_addr_LO); 795 796 } else if (src->is_single_fpu()) { 797 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix()); 798 __ strs(src->as_float_reg(), dest_addr); 799 800 } else if (src->is_double_fpu()) { 801 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix()); 802 __ strd(src->as_double_reg(), dest_addr); 803 804 } else { 805 ShouldNotReachHere(); 806 } 807 808 } 809 810 811 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) { 812 LIR_Address* to_addr = dest->as_address_ptr(); 813 PatchingStub* patch = NULL; 814 Register compressed_src = rscratch1; 815 816 if (patch_code != lir_patch_none) { 817 deoptimize_trap(info); 818 return; 819 } 820 821 if (type == T_ARRAY || type == T_OBJECT) { 822 __ verify_oop(src->as_register()); 823 824 if (UseCompressedOops && !wide) { 825 __ encode_heap_oop(compressed_src, src->as_register()); 826 } else { 827 compressed_src = src->as_register(); 828 } 829 } 830 831 int null_check_here = code_offset(); 832 switch (type) { 833 case T_FLOAT: { 834 __ strs(src->as_float_reg(), as_Address(to_addr)); 835 break; 836 } 837 838 case T_DOUBLE: { 839 __ strd(src->as_double_reg(), as_Address(to_addr)); 840 break; 841 } 842 843 case T_ARRAY: // fall through 844 case T_OBJECT: // fall through 845 if (UseCompressedOops && !wide) { 846 __ strw(compressed_src, as_Address(to_addr, rscratch2)); 847 } else { 848 __ str(compressed_src, as_Address(to_addr)); 849 } 850 break; 851 case T_METADATA: 852 // We get here to store a method pointer to the stack to pass to 853 // a dtrace runtime call. This can't work on 64 bit with 854 // compressed klass ptrs: T_METADATA can be a compressed klass 855 // ptr or a 64 bit method pointer. 856 ShouldNotReachHere(); 857 __ str(src->as_register(), as_Address(to_addr)); 858 break; 859 case T_ADDRESS: 860 __ str(src->as_register(), as_Address(to_addr)); 861 break; 862 case T_INT: 863 __ strw(src->as_register(), as_Address(to_addr)); 864 break; 865 866 case T_LONG: { 867 __ str(src->as_register_lo(), as_Address_lo(to_addr)); 868 break; 869 } 870 871 case T_BYTE: // fall through 872 case T_BOOLEAN: { 873 __ strb(src->as_register(), as_Address(to_addr)); 874 break; 875 } 876 877 case T_CHAR: // fall through 878 case T_SHORT: 879 __ strh(src->as_register(), as_Address(to_addr)); 880 break; 881 882 default: 883 ShouldNotReachHere(); 884 } 885 if (info != NULL) { 886 add_debug_info_for_null_check(null_check_here, info); 887 } 888 } 889 890 891 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 892 assert(src->is_stack(), "should not call otherwise"); 893 assert(dest->is_register(), "should not call otherwise"); 894 895 if (dest->is_single_cpu()) { 896 if (type == T_ARRAY || type == T_OBJECT) { 897 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 898 __ verify_oop(dest->as_register()); 899 } else if (type == T_METADATA) { 900 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 901 } else { 902 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 903 } 904 905 } else if (dest->is_double_cpu()) { 906 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes); 907 __ ldr(dest->as_register_lo(), src_addr_LO); 908 909 } else if (dest->is_single_fpu()) { 910 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix()); 911 __ ldrs(dest->as_float_reg(), src_addr); 912 913 } else if (dest->is_double_fpu()) { 914 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix()); 915 __ ldrd(dest->as_double_reg(), src_addr); 916 917 } else { 918 ShouldNotReachHere(); 919 } 920 } 921 922 923 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 924 address target = NULL; 925 relocInfo::relocType reloc_type = relocInfo::none; 926 927 switch (patching_id(info)) { 928 case PatchingStub::access_field_id: 929 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 930 reloc_type = relocInfo::section_word_type; 931 break; 932 case PatchingStub::load_klass_id: 933 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 934 reloc_type = relocInfo::metadata_type; 935 break; 936 case PatchingStub::load_mirror_id: 937 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 938 reloc_type = relocInfo::oop_type; 939 break; 940 case PatchingStub::load_appendix_id: 941 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 942 reloc_type = relocInfo::oop_type; 943 break; 944 default: ShouldNotReachHere(); 945 } 946 947 __ far_call(RuntimeAddress(target)); 948 add_call_info_here(info); 949 } 950 951 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 952 953 LIR_Opr temp; 954 if (type == T_LONG || type == T_DOUBLE) 955 temp = FrameMap::rscratch1_long_opr; 956 else 957 temp = FrameMap::rscratch1_opr; 958 959 stack2reg(src, temp, src->type()); 960 reg2stack(temp, dest, dest->type(), false); 961 } 962 963 964 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) { 965 LIR_Address* addr = src->as_address_ptr(); 966 LIR_Address* from_addr = src->as_address_ptr(); 967 968 if (addr->base()->type() == T_OBJECT) { 969 __ verify_oop(addr->base()->as_pointer_register()); 970 } 971 972 if (patch_code != lir_patch_none) { 973 deoptimize_trap(info); 974 return; 975 } 976 977 if (info != NULL) { 978 add_debug_info_for_null_check_here(info); 979 } 980 int null_check_here = code_offset(); 981 switch (type) { 982 case T_FLOAT: { 983 __ ldrs(dest->as_float_reg(), as_Address(from_addr)); 984 break; 985 } 986 987 case T_DOUBLE: { 988 __ ldrd(dest->as_double_reg(), as_Address(from_addr)); 989 break; 990 } 991 992 case T_ARRAY: // fall through 993 case T_OBJECT: // fall through 994 if (UseCompressedOops && !wide) { 995 __ ldrw(dest->as_register(), as_Address(from_addr)); 996 } else { 997 __ ldr(dest->as_register(), as_Address(from_addr)); 998 } 999 break; 1000 case T_METADATA: 1001 // We get here to store a method pointer to the stack to pass to 1002 // a dtrace runtime call. This can't work on 64 bit with 1003 // compressed klass ptrs: T_METADATA can be a compressed klass 1004 // ptr or a 64 bit method pointer. 1005 ShouldNotReachHere(); 1006 __ ldr(dest->as_register(), as_Address(from_addr)); 1007 break; 1008 case T_ADDRESS: 1009 // FIXME: OMG this is a horrible kludge. Any offset from an 1010 // address that matches klass_offset_in_bytes() will be loaded 1011 // as a word, not a long. 1012 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1013 __ ldrw(dest->as_register(), as_Address(from_addr)); 1014 } else { 1015 __ ldr(dest->as_register(), as_Address(from_addr)); 1016 } 1017 break; 1018 case T_INT: 1019 __ ldrw(dest->as_register(), as_Address(from_addr)); 1020 break; 1021 1022 case T_LONG: { 1023 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr)); 1024 break; 1025 } 1026 1027 case T_BYTE: 1028 __ ldrsb(dest->as_register(), as_Address(from_addr)); 1029 break; 1030 case T_BOOLEAN: { 1031 __ ldrb(dest->as_register(), as_Address(from_addr)); 1032 break; 1033 } 1034 1035 case T_CHAR: 1036 __ ldrh(dest->as_register(), as_Address(from_addr)); 1037 break; 1038 case T_SHORT: 1039 __ ldrsh(dest->as_register(), as_Address(from_addr)); 1040 break; 1041 1042 default: 1043 ShouldNotReachHere(); 1044 } 1045 1046 if (type == T_ARRAY || type == T_OBJECT) { 1047 if (UseCompressedOops && !wide) { 1048 __ decode_heap_oop(dest->as_register()); 1049 } 1050 __ verify_oop(dest->as_register()); 1051 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1052 if (UseCompressedClassPointers) { 1053 __ decode_klass_not_null(dest->as_register()); 1054 } 1055 } 1056 } 1057 1058 1059 int LIR_Assembler::array_element_size(BasicType type) const { 1060 int elem_size = type2aelembytes(type); 1061 return exact_log2(elem_size); 1062 } 1063 1064 void LIR_Assembler::arithmetic_idiv(LIR_Op3* op, bool is_irem) { 1065 Register Rdividend = op->in_opr1()->as_register(); 1066 Register Rdivisor = op->in_opr2()->as_register(); 1067 Register Rscratch = op->in_opr3()->as_register(); 1068 Register Rresult = op->result_opr()->as_register(); 1069 int divisor = -1; 1070 1071 /* 1072 TODO: For some reason, using the Rscratch that gets passed in is 1073 not possible because the register allocator does not see the tmp reg 1074 as used, and assignes it the same register as Rdividend. We use rscratch1 1075 instead. 1076 1077 assert(Rdividend != Rscratch, ""); 1078 assert(Rdivisor != Rscratch, ""); 1079 */ 1080 1081 if (Rdivisor == noreg && is_power_of_2(divisor)) { 1082 // convert division by a power of two into some shifts and logical operations 1083 } 1084 1085 __ corrected_idivl(Rresult, Rdividend, Rdivisor, is_irem, rscratch1); 1086 } 1087 1088 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1089 switch (op->code()) { 1090 case lir_idiv: 1091 arithmetic_idiv(op, false); 1092 break; 1093 case lir_irem: 1094 arithmetic_idiv(op, true); 1095 break; 1096 case lir_fmad: 1097 __ fmaddd(op->result_opr()->as_double_reg(), 1098 op->in_opr1()->as_double_reg(), 1099 op->in_opr2()->as_double_reg(), 1100 op->in_opr3()->as_double_reg()); 1101 break; 1102 case lir_fmaf: 1103 __ fmadds(op->result_opr()->as_float_reg(), 1104 op->in_opr1()->as_float_reg(), 1105 op->in_opr2()->as_float_reg(), 1106 op->in_opr3()->as_float_reg()); 1107 break; 1108 default: ShouldNotReachHere(); break; 1109 } 1110 } 1111 1112 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1113 #ifdef ASSERT 1114 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label"); 1115 if (op->block() != NULL) _branch_target_blocks.append(op->block()); 1116 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock()); 1117 #endif 1118 1119 if (op->cond() == lir_cond_always) { 1120 if (op->info() != NULL) add_debug_info_for_branch(op->info()); 1121 __ b(*(op->label())); 1122 } else { 1123 Assembler::Condition acond; 1124 if (op->code() == lir_cond_float_branch) { 1125 bool is_unordered = (op->ublock() == op->block()); 1126 // Assembler::EQ does not permit unordered branches, so we add 1127 // another branch here. Likewise, Assembler::NE does not permit 1128 // ordered branches. 1129 if (is_unordered && op->cond() == lir_cond_equal 1130 || !is_unordered && op->cond() == lir_cond_notEqual) 1131 __ br(Assembler::VS, *(op->ublock()->label())); 1132 switch(op->cond()) { 1133 case lir_cond_equal: acond = Assembler::EQ; break; 1134 case lir_cond_notEqual: acond = Assembler::NE; break; 1135 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break; 1136 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break; 1137 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break; 1138 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break; 1139 default: ShouldNotReachHere(); 1140 acond = Assembler::EQ; // unreachable 1141 } 1142 } else { 1143 switch (op->cond()) { 1144 case lir_cond_equal: acond = Assembler::EQ; break; 1145 case lir_cond_notEqual: acond = Assembler::NE; break; 1146 case lir_cond_less: acond = Assembler::LT; break; 1147 case lir_cond_lessEqual: acond = Assembler::LE; break; 1148 case lir_cond_greaterEqual: acond = Assembler::GE; break; 1149 case lir_cond_greater: acond = Assembler::GT; break; 1150 case lir_cond_belowEqual: acond = Assembler::LS; break; 1151 case lir_cond_aboveEqual: acond = Assembler::HS; break; 1152 default: ShouldNotReachHere(); 1153 acond = Assembler::EQ; // unreachable 1154 } 1155 } 1156 __ br(acond,*(op->label())); 1157 } 1158 } 1159 1160 1161 1162 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1163 LIR_Opr src = op->in_opr(); 1164 LIR_Opr dest = op->result_opr(); 1165 1166 switch (op->bytecode()) { 1167 case Bytecodes::_i2f: 1168 { 1169 __ scvtfws(dest->as_float_reg(), src->as_register()); 1170 break; 1171 } 1172 case Bytecodes::_i2d: 1173 { 1174 __ scvtfwd(dest->as_double_reg(), src->as_register()); 1175 break; 1176 } 1177 case Bytecodes::_l2d: 1178 { 1179 __ scvtfd(dest->as_double_reg(), src->as_register_lo()); 1180 break; 1181 } 1182 case Bytecodes::_l2f: 1183 { 1184 __ scvtfs(dest->as_float_reg(), src->as_register_lo()); 1185 break; 1186 } 1187 case Bytecodes::_f2d: 1188 { 1189 __ fcvts(dest->as_double_reg(), src->as_float_reg()); 1190 break; 1191 } 1192 case Bytecodes::_d2f: 1193 { 1194 __ fcvtd(dest->as_float_reg(), src->as_double_reg()); 1195 break; 1196 } 1197 case Bytecodes::_i2c: 1198 { 1199 __ ubfx(dest->as_register(), src->as_register(), 0, 16); 1200 break; 1201 } 1202 case Bytecodes::_i2l: 1203 { 1204 __ sxtw(dest->as_register_lo(), src->as_register()); 1205 break; 1206 } 1207 case Bytecodes::_i2s: 1208 { 1209 __ sxth(dest->as_register(), src->as_register()); 1210 break; 1211 } 1212 case Bytecodes::_i2b: 1213 { 1214 __ sxtb(dest->as_register(), src->as_register()); 1215 break; 1216 } 1217 case Bytecodes::_l2i: 1218 { 1219 _masm->block_comment("FIXME: This could be a no-op"); 1220 __ uxtw(dest->as_register(), src->as_register_lo()); 1221 break; 1222 } 1223 case Bytecodes::_d2l: 1224 { 1225 __ fcvtzd(dest->as_register_lo(), src->as_double_reg()); 1226 break; 1227 } 1228 case Bytecodes::_f2i: 1229 { 1230 __ fcvtzsw(dest->as_register(), src->as_float_reg()); 1231 break; 1232 } 1233 case Bytecodes::_f2l: 1234 { 1235 __ fcvtzs(dest->as_register_lo(), src->as_float_reg()); 1236 break; 1237 } 1238 case Bytecodes::_d2i: 1239 { 1240 __ fcvtzdw(dest->as_register(), src->as_double_reg()); 1241 break; 1242 } 1243 default: ShouldNotReachHere(); 1244 } 1245 } 1246 1247 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1248 if (op->init_check()) { 1249 __ ldrb(rscratch1, Address(op->klass()->as_register(), 1250 InstanceKlass::init_state_offset())); 1251 __ cmpw(rscratch1, InstanceKlass::fully_initialized); 1252 add_debug_info_for_null_check_here(op->stub()->info()); 1253 __ br(Assembler::NE, *op->stub()->entry()); 1254 } 1255 __ allocate_object(op->obj()->as_register(), 1256 op->tmp1()->as_register(), 1257 op->tmp2()->as_register(), 1258 op->header_size(), 1259 op->object_size(), 1260 op->klass()->as_register(), 1261 *op->stub()->entry()); 1262 __ bind(*op->stub()->continuation()); 1263 } 1264 1265 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1266 Register len = op->len()->as_register(); 1267 __ uxtw(len, len); 1268 1269 if (UseSlowPath || 1270 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || 1271 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { 1272 __ b(*op->stub()->entry()); 1273 } else { 1274 Register tmp1 = op->tmp1()->as_register(); 1275 Register tmp2 = op->tmp2()->as_register(); 1276 Register tmp3 = op->tmp3()->as_register(); 1277 if (len == tmp1) { 1278 tmp1 = tmp3; 1279 } else if (len == tmp2) { 1280 tmp2 = tmp3; 1281 } else if (len == tmp3) { 1282 // everything is ok 1283 } else { 1284 __ mov(tmp3, len); 1285 } 1286 __ allocate_array(op->obj()->as_register(), 1287 len, 1288 tmp1, 1289 tmp2, 1290 arrayOopDesc::header_size(op->type()), 1291 array_element_size(op->type()), 1292 op->klass()->as_register(), 1293 *op->stub()->entry()); 1294 } 1295 __ bind(*op->stub()->continuation()); 1296 } 1297 1298 void LIR_Assembler::type_profile_helper(Register mdo, 1299 ciMethodData *md, ciProfileData *data, 1300 Register recv, Label* update_done) { 1301 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1302 Label next_test; 1303 // See if the receiver is receiver[n]. 1304 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1305 __ ldr(rscratch1, Address(rscratch2)); 1306 __ cmp(recv, rscratch1); 1307 __ br(Assembler::NE, next_test); 1308 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); 1309 __ addptr(data_addr, DataLayout::counter_increment); 1310 __ b(*update_done); 1311 __ bind(next_test); 1312 } 1313 1314 // Didn't find receiver; find next empty slot and fill it in 1315 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1316 Label next_test; 1317 __ lea(rscratch2, 1318 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1319 Address recv_addr(rscratch2); 1320 __ ldr(rscratch1, recv_addr); 1321 __ cbnz(rscratch1, next_test); 1322 __ str(recv, recv_addr); 1323 __ mov(rscratch1, DataLayout::counter_increment); 1324 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)))); 1325 __ str(rscratch1, Address(rscratch2)); 1326 __ b(*update_done); 1327 __ bind(next_test); 1328 } 1329 } 1330 1331 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1332 // we always need a stub for the failure case. 1333 CodeStub* stub = op->stub(); 1334 Register obj = op->object()->as_register(); 1335 Register k_RInfo = op->tmp1()->as_register(); 1336 Register klass_RInfo = op->tmp2()->as_register(); 1337 Register dst = op->result_opr()->as_register(); 1338 ciKlass* k = op->klass(); 1339 Register Rtmp1 = noreg; 1340 1341 // check if it needs to be profiled 1342 ciMethodData* md; 1343 ciProfileData* data; 1344 1345 const bool should_profile = op->should_profile(); 1346 1347 if (should_profile) { 1348 ciMethod* method = op->profiled_method(); 1349 assert(method != NULL, "Should have method"); 1350 int bci = op->profiled_bci(); 1351 md = method->method_data_or_null(); 1352 assert(md != NULL, "Sanity"); 1353 data = md->bci_to_data(bci); 1354 assert(data != NULL, "need data for type check"); 1355 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1356 } 1357 Label profile_cast_success, profile_cast_failure; 1358 Label *success_target = should_profile ? &profile_cast_success : success; 1359 Label *failure_target = should_profile ? &profile_cast_failure : failure; 1360 1361 if (obj == k_RInfo) { 1362 k_RInfo = dst; 1363 } else if (obj == klass_RInfo) { 1364 klass_RInfo = dst; 1365 } 1366 if (k->is_loaded() && !UseCompressedClassPointers) { 1367 select_different_registers(obj, dst, k_RInfo, klass_RInfo); 1368 } else { 1369 Rtmp1 = op->tmp3()->as_register(); 1370 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1371 } 1372 1373 assert_different_registers(obj, k_RInfo, klass_RInfo); 1374 1375 if (should_profile) { 1376 Label not_null; 1377 __ cbnz(obj, not_null); 1378 // Object is null; update MDO and exit 1379 Register mdo = klass_RInfo; 1380 __ mov_metadata(mdo, md->constant_encoding()); 1381 Address data_addr 1382 = __ form_address(rscratch2, mdo, 1383 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()), 1384 LogBytesPerWord); 1385 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant()); 1386 __ ldr(rscratch1, data_addr); 1387 __ orr(rscratch1, rscratch1, header_bits); 1388 __ str(rscratch1, data_addr); 1389 __ b(*obj_is_null); 1390 __ bind(not_null); 1391 } else { 1392 __ cbz(obj, *obj_is_null); 1393 } 1394 1395 if (!k->is_loaded()) { 1396 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1397 } else { 1398 __ mov_metadata(k_RInfo, k->constant_encoding()); 1399 } 1400 __ verify_oop(obj); 1401 1402 if (op->fast_check()) { 1403 // get object class 1404 // not a safepoint as obj null check happens earlier 1405 __ load_klass(rscratch1, obj); 1406 __ cmp( rscratch1, k_RInfo); 1407 1408 __ br(Assembler::NE, *failure_target); 1409 // successful cast, fall through to profile or jump 1410 } else { 1411 // get object class 1412 // not a safepoint as obj null check happens earlier 1413 __ load_klass(klass_RInfo, obj); 1414 if (k->is_loaded()) { 1415 // See if we get an immediate positive hit 1416 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset()))); 1417 __ cmp(k_RInfo, rscratch1); 1418 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1419 __ br(Assembler::NE, *failure_target); 1420 // successful cast, fall through to profile or jump 1421 } else { 1422 // See if we get an immediate positive hit 1423 __ br(Assembler::EQ, *success_target); 1424 // check for self 1425 __ cmp(klass_RInfo, k_RInfo); 1426 __ br(Assembler::EQ, *success_target); 1427 1428 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1429 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1430 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1431 // result is a boolean 1432 __ cbzw(klass_RInfo, *failure_target); 1433 // successful cast, fall through to profile or jump 1434 } 1435 } else { 1436 // perform the fast part of the checking logic 1437 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1438 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1439 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1440 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1441 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1442 // result is a boolean 1443 __ cbz(k_RInfo, *failure_target); 1444 // successful cast, fall through to profile or jump 1445 } 1446 } 1447 if (should_profile) { 1448 Register mdo = klass_RInfo, recv = k_RInfo; 1449 __ bind(profile_cast_success); 1450 __ mov_metadata(mdo, md->constant_encoding()); 1451 __ load_klass(recv, obj); 1452 Label update_done; 1453 type_profile_helper(mdo, md, data, recv, success); 1454 __ b(*success); 1455 1456 __ bind(profile_cast_failure); 1457 __ mov_metadata(mdo, md->constant_encoding()); 1458 Address counter_addr 1459 = __ form_address(rscratch2, mdo, 1460 md->byte_offset_of_slot(data, CounterData::count_offset()), 1461 LogBytesPerWord); 1462 __ ldr(rscratch1, counter_addr); 1463 __ sub(rscratch1, rscratch1, DataLayout::counter_increment); 1464 __ str(rscratch1, counter_addr); 1465 __ b(*failure); 1466 } 1467 __ b(*success); 1468 } 1469 1470 1471 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1472 const bool should_profile = op->should_profile(); 1473 1474 LIR_Code code = op->code(); 1475 if (code == lir_store_check) { 1476 Register value = op->object()->as_register(); 1477 Register array = op->array()->as_register(); 1478 Register k_RInfo = op->tmp1()->as_register(); 1479 Register klass_RInfo = op->tmp2()->as_register(); 1480 Register Rtmp1 = op->tmp3()->as_register(); 1481 1482 CodeStub* stub = op->stub(); 1483 1484 // check if it needs to be profiled 1485 ciMethodData* md; 1486 ciProfileData* data; 1487 1488 if (should_profile) { 1489 ciMethod* method = op->profiled_method(); 1490 assert(method != NULL, "Should have method"); 1491 int bci = op->profiled_bci(); 1492 md = method->method_data_or_null(); 1493 assert(md != NULL, "Sanity"); 1494 data = md->bci_to_data(bci); 1495 assert(data != NULL, "need data for type check"); 1496 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1497 } 1498 Label profile_cast_success, profile_cast_failure, done; 1499 Label *success_target = should_profile ? &profile_cast_success : &done; 1500 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry(); 1501 1502 if (should_profile) { 1503 Label not_null; 1504 __ cbnz(value, not_null); 1505 // Object is null; update MDO and exit 1506 Register mdo = klass_RInfo; 1507 __ mov_metadata(mdo, md->constant_encoding()); 1508 Address data_addr 1509 = __ form_address(rscratch2, mdo, 1510 md->byte_offset_of_slot(data, DataLayout::header_offset()), 1511 LogBytesPerInt); 1512 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant()); 1513 __ ldrw(rscratch1, data_addr); 1514 __ orrw(rscratch1, rscratch1, header_bits); 1515 __ strw(rscratch1, data_addr); 1516 __ b(done); 1517 __ bind(not_null); 1518 } else { 1519 __ cbz(value, done); 1520 } 1521 1522 add_debug_info_for_null_check_here(op->info_for_exception()); 1523 __ load_klass(k_RInfo, array); 1524 __ load_klass(klass_RInfo, value); 1525 1526 // get instance klass (it's already uncompressed) 1527 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1528 // perform the fast part of the checking logic 1529 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1530 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1531 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1532 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1533 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1534 // result is a boolean 1535 __ cbzw(k_RInfo, *failure_target); 1536 // fall through to the success case 1537 1538 if (should_profile) { 1539 Register mdo = klass_RInfo, recv = k_RInfo; 1540 __ bind(profile_cast_success); 1541 __ mov_metadata(mdo, md->constant_encoding()); 1542 __ load_klass(recv, value); 1543 Label update_done; 1544 type_profile_helper(mdo, md, data, recv, &done); 1545 __ b(done); 1546 1547 __ bind(profile_cast_failure); 1548 __ mov_metadata(mdo, md->constant_encoding()); 1549 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1550 __ lea(rscratch2, counter_addr); 1551 __ ldr(rscratch1, Address(rscratch2)); 1552 __ sub(rscratch1, rscratch1, DataLayout::counter_increment); 1553 __ str(rscratch1, Address(rscratch2)); 1554 __ b(*stub->entry()); 1555 } 1556 1557 __ bind(done); 1558 } else if (code == lir_checkcast) { 1559 Register obj = op->object()->as_register(); 1560 Register dst = op->result_opr()->as_register(); 1561 Label success; 1562 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1563 __ bind(success); 1564 if (dst != obj) { 1565 __ mov(dst, obj); 1566 } 1567 } else if (code == lir_instanceof) { 1568 Register obj = op->object()->as_register(); 1569 Register dst = op->result_opr()->as_register(); 1570 Label success, failure, done; 1571 emit_typecheck_helper(op, &success, &failure, &failure); 1572 __ bind(failure); 1573 __ mov(dst, zr); 1574 __ b(done); 1575 __ bind(success); 1576 __ mov(dst, 1); 1577 __ bind(done); 1578 } else { 1579 ShouldNotReachHere(); 1580 } 1581 } 1582 1583 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { 1584 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); 1585 __ cset(rscratch1, Assembler::NE); 1586 __ membar(__ AnyAny); 1587 } 1588 1589 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { 1590 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); 1591 __ cset(rscratch1, Assembler::NE); 1592 __ membar(__ AnyAny); 1593 } 1594 1595 1596 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1597 assert(VM_Version::supports_cx8(), "wrong machine"); 1598 Register addr; 1599 if (op->addr()->is_register()) { 1600 addr = as_reg(op->addr()); 1601 } else { 1602 assert(op->addr()->is_address(), "what else?"); 1603 LIR_Address* addr_ptr = op->addr()->as_address_ptr(); 1604 assert(addr_ptr->disp() == 0, "need 0 disp"); 1605 assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index"); 1606 addr = as_reg(addr_ptr->base()); 1607 } 1608 Register newval = as_reg(op->new_value()); 1609 Register cmpval = as_reg(op->cmp_value()); 1610 Label succeed, fail, around; 1611 1612 if (op->code() == lir_cas_obj) { 1613 if (UseCompressedOops) { 1614 Register t1 = op->tmp1()->as_register(); 1615 assert(op->tmp1()->is_valid(), "must be"); 1616 __ encode_heap_oop(t1, cmpval); 1617 cmpval = t1; 1618 __ encode_heap_oop(rscratch2, newval); 1619 newval = rscratch2; 1620 casw(addr, newval, cmpval); 1621 } else { 1622 casl(addr, newval, cmpval); 1623 } 1624 } else if (op->code() == lir_cas_int) { 1625 casw(addr, newval, cmpval); 1626 } else { 1627 casl(addr, newval, cmpval); 1628 } 1629 } 1630 1631 1632 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) { 1633 1634 Assembler::Condition acond, ncond; 1635 switch (condition) { 1636 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break; 1637 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break; 1638 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break; 1639 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break; 1640 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break; 1641 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break; 1642 case lir_cond_belowEqual: 1643 case lir_cond_aboveEqual: 1644 default: ShouldNotReachHere(); 1645 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable 1646 } 1647 1648 assert(result->is_single_cpu() || result->is_double_cpu(), 1649 "expect single register for result"); 1650 if (opr1->is_constant() && opr2->is_constant() 1651 && opr1->type() == T_INT && opr2->type() == T_INT) { 1652 jint val1 = opr1->as_jint(); 1653 jint val2 = opr2->as_jint(); 1654 if (val1 == 0 && val2 == 1) { 1655 __ cset(result->as_register(), ncond); 1656 return; 1657 } else if (val1 == 1 && val2 == 0) { 1658 __ cset(result->as_register(), acond); 1659 return; 1660 } 1661 } 1662 1663 if (opr1->is_constant() && opr2->is_constant() 1664 && opr1->type() == T_LONG && opr2->type() == T_LONG) { 1665 jlong val1 = opr1->as_jlong(); 1666 jlong val2 = opr2->as_jlong(); 1667 if (val1 == 0 && val2 == 1) { 1668 __ cset(result->as_register_lo(), ncond); 1669 return; 1670 } else if (val1 == 1 && val2 == 0) { 1671 __ cset(result->as_register_lo(), acond); 1672 return; 1673 } 1674 } 1675 1676 if (opr1->is_stack()) { 1677 stack2reg(opr1, FrameMap::rscratch1_opr, result->type()); 1678 opr1 = FrameMap::rscratch1_opr; 1679 } else if (opr1->is_constant()) { 1680 LIR_Opr tmp 1681 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr; 1682 const2reg(opr1, tmp, lir_patch_none, NULL); 1683 opr1 = tmp; 1684 } 1685 1686 if (opr2->is_stack()) { 1687 stack2reg(opr2, FrameMap::rscratch2_opr, result->type()); 1688 opr2 = FrameMap::rscratch2_opr; 1689 } else if (opr2->is_constant()) { 1690 LIR_Opr tmp 1691 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr; 1692 const2reg(opr2, tmp, lir_patch_none, NULL); 1693 opr2 = tmp; 1694 } 1695 1696 if (result->type() == T_LONG) 1697 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond); 1698 else 1699 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond); 1700 } 1701 1702 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 1703 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method"); 1704 1705 if (left->is_single_cpu()) { 1706 Register lreg = left->as_register(); 1707 Register dreg = as_reg(dest); 1708 1709 if (right->is_single_cpu()) { 1710 // cpu register - cpu register 1711 1712 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT, 1713 "should be"); 1714 Register rreg = right->as_register(); 1715 switch (code) { 1716 case lir_add: __ addw (dest->as_register(), lreg, rreg); break; 1717 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break; 1718 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break; 1719 default: ShouldNotReachHere(); 1720 } 1721 1722 } else if (right->is_double_cpu()) { 1723 Register rreg = right->as_register_lo(); 1724 // single_cpu + double_cpu: can happen with obj+long 1725 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1726 switch (code) { 1727 case lir_add: __ add(dreg, lreg, rreg); break; 1728 case lir_sub: __ sub(dreg, lreg, rreg); break; 1729 default: ShouldNotReachHere(); 1730 } 1731 } else if (right->is_constant()) { 1732 // cpu register - constant 1733 jlong c; 1734 1735 // FIXME. This is fugly: we really need to factor all this logic. 1736 switch(right->type()) { 1737 case T_LONG: 1738 c = right->as_constant_ptr()->as_jlong(); 1739 break; 1740 case T_INT: 1741 case T_ADDRESS: 1742 c = right->as_constant_ptr()->as_jint(); 1743 break; 1744 default: 1745 ShouldNotReachHere(); 1746 c = 0; // unreachable 1747 break; 1748 } 1749 1750 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1751 if (c == 0 && dreg == lreg) { 1752 COMMENT("effective nop elided"); 1753 return; 1754 } 1755 switch(left->type()) { 1756 case T_INT: 1757 switch (code) { 1758 case lir_add: __ addw(dreg, lreg, c); break; 1759 case lir_sub: __ subw(dreg, lreg, c); break; 1760 default: ShouldNotReachHere(); 1761 } 1762 break; 1763 case T_OBJECT: 1764 case T_ADDRESS: 1765 switch (code) { 1766 case lir_add: __ add(dreg, lreg, c); break; 1767 case lir_sub: __ sub(dreg, lreg, c); break; 1768 default: ShouldNotReachHere(); 1769 } 1770 break; 1771 ShouldNotReachHere(); 1772 } 1773 } else { 1774 ShouldNotReachHere(); 1775 } 1776 1777 } else if (left->is_double_cpu()) { 1778 Register lreg_lo = left->as_register_lo(); 1779 1780 if (right->is_double_cpu()) { 1781 // cpu register - cpu register 1782 Register rreg_lo = right->as_register_lo(); 1783 switch (code) { 1784 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1785 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1786 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1787 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break; 1788 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break; 1789 default: 1790 ShouldNotReachHere(); 1791 } 1792 1793 } else if (right->is_constant()) { 1794 jlong c = right->as_constant_ptr()->as_jlong_bits(); 1795 Register dreg = as_reg(dest); 1796 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1797 if (c == 0 && dreg == lreg_lo) { 1798 COMMENT("effective nop elided"); 1799 return; 1800 } 1801 switch (code) { 1802 case lir_add: __ add(dreg, lreg_lo, c); break; 1803 case lir_sub: __ sub(dreg, lreg_lo, c); break; 1804 default: 1805 ShouldNotReachHere(); 1806 } 1807 } else { 1808 ShouldNotReachHere(); 1809 } 1810 } else if (left->is_single_fpu()) { 1811 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register"); 1812 switch (code) { 1813 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1814 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1815 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1816 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1817 default: 1818 ShouldNotReachHere(); 1819 } 1820 } else if (left->is_double_fpu()) { 1821 if (right->is_double_fpu()) { 1822 // cpu register - cpu register 1823 switch (code) { 1824 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1825 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1826 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1827 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1828 default: 1829 ShouldNotReachHere(); 1830 } 1831 } else { 1832 if (right->is_constant()) { 1833 ShouldNotReachHere(); 1834 } 1835 ShouldNotReachHere(); 1836 } 1837 } else if (left->is_single_stack() || left->is_address()) { 1838 assert(left == dest, "left and dest must be equal"); 1839 ShouldNotReachHere(); 1840 } else { 1841 ShouldNotReachHere(); 1842 } 1843 } 1844 1845 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); } 1846 1847 1848 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { 1849 switch(code) { 1850 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break; 1851 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break; 1852 default : ShouldNotReachHere(); 1853 } 1854 } 1855 1856 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 1857 1858 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register"); 1859 Register Rleft = left->is_single_cpu() ? left->as_register() : 1860 left->as_register_lo(); 1861 if (dst->is_single_cpu()) { 1862 Register Rdst = dst->as_register(); 1863 if (right->is_constant()) { 1864 switch (code) { 1865 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break; 1866 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break; 1867 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break; 1868 default: ShouldNotReachHere(); break; 1869 } 1870 } else { 1871 Register Rright = right->is_single_cpu() ? right->as_register() : 1872 right->as_register_lo(); 1873 switch (code) { 1874 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break; 1875 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break; 1876 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break; 1877 default: ShouldNotReachHere(); break; 1878 } 1879 } 1880 } else { 1881 Register Rdst = dst->as_register_lo(); 1882 if (right->is_constant()) { 1883 switch (code) { 1884 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break; 1885 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break; 1886 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break; 1887 default: ShouldNotReachHere(); break; 1888 } 1889 } else { 1890 Register Rright = right->is_single_cpu() ? right->as_register() : 1891 right->as_register_lo(); 1892 switch (code) { 1893 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break; 1894 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break; 1895 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break; 1896 default: ShouldNotReachHere(); break; 1897 } 1898 } 1899 } 1900 } 1901 1902 1903 1904 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); } 1905 1906 1907 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 1908 if (opr1->is_constant() && opr2->is_single_cpu()) { 1909 // tableswitch 1910 Register reg = as_reg(opr2); 1911 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()]; 1912 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after); 1913 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) { 1914 Register reg1 = as_reg(opr1); 1915 if (opr2->is_single_cpu()) { 1916 // cpu register - cpu register 1917 Register reg2 = opr2->as_register(); 1918 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) { 1919 __ cmp(reg1, reg2); 1920 } else { 1921 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?"); 1922 __ cmpw(reg1, reg2); 1923 } 1924 return; 1925 } 1926 if (opr2->is_double_cpu()) { 1927 // cpu register - cpu register 1928 Register reg2 = opr2->as_register_lo(); 1929 __ cmp(reg1, reg2); 1930 return; 1931 } 1932 1933 if (opr2->is_constant()) { 1934 bool is_32bit = false; // width of register operand 1935 jlong imm; 1936 1937 switch(opr2->type()) { 1938 case T_INT: 1939 imm = opr2->as_constant_ptr()->as_jint(); 1940 is_32bit = true; 1941 break; 1942 case T_LONG: 1943 imm = opr2->as_constant_ptr()->as_jlong(); 1944 break; 1945 case T_ADDRESS: 1946 imm = opr2->as_constant_ptr()->as_jint(); 1947 break; 1948 case T_OBJECT: 1949 case T_ARRAY: 1950 imm = jlong(opr2->as_constant_ptr()->as_jobject()); 1951 break; 1952 default: 1953 ShouldNotReachHere(); 1954 imm = 0; // unreachable 1955 break; 1956 } 1957 1958 if (Assembler::operand_valid_for_add_sub_immediate(imm)) { 1959 if (is_32bit) 1960 __ cmpw(reg1, imm); 1961 else 1962 __ cmp(reg1, imm); 1963 return; 1964 } else { 1965 __ mov(rscratch1, imm); 1966 if (is_32bit) 1967 __ cmpw(reg1, rscratch1); 1968 else 1969 __ cmp(reg1, rscratch1); 1970 return; 1971 } 1972 } else 1973 ShouldNotReachHere(); 1974 } else if (opr1->is_single_fpu()) { 1975 FloatRegister reg1 = opr1->as_float_reg(); 1976 assert(opr2->is_single_fpu(), "expect single float register"); 1977 FloatRegister reg2 = opr2->as_float_reg(); 1978 __ fcmps(reg1, reg2); 1979 } else if (opr1->is_double_fpu()) { 1980 FloatRegister reg1 = opr1->as_double_reg(); 1981 assert(opr2->is_double_fpu(), "expect double float register"); 1982 FloatRegister reg2 = opr2->as_double_reg(); 1983 __ fcmpd(reg1, reg2); 1984 } else { 1985 ShouldNotReachHere(); 1986 } 1987 } 1988 1989 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){ 1990 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 1991 bool is_unordered_less = (code == lir_ucmp_fd2i); 1992 if (left->is_single_fpu()) { 1993 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register()); 1994 } else if (left->is_double_fpu()) { 1995 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register()); 1996 } else { 1997 ShouldNotReachHere(); 1998 } 1999 } else if (code == lir_cmp_l2i) { 2000 Label done; 2001 __ cmp(left->as_register_lo(), right->as_register_lo()); 2002 __ mov(dst->as_register(), (u_int64_t)-1L); 2003 __ br(Assembler::LT, done); 2004 __ csinc(dst->as_register(), zr, zr, Assembler::EQ); 2005 __ bind(done); 2006 } else { 2007 ShouldNotReachHere(); 2008 } 2009 } 2010 2011 2012 void LIR_Assembler::align_call(LIR_Code code) { } 2013 2014 2015 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 2016 address call = __ trampoline_call(Address(op->addr(), rtype)); 2017 if (call == NULL) { 2018 bailout("trampoline stub overflow"); 2019 return; 2020 } 2021 add_call_info(code_offset(), op->info()); 2022 } 2023 2024 2025 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 2026 address call = __ ic_call(op->addr()); 2027 if (call == NULL) { 2028 bailout("trampoline stub overflow"); 2029 return; 2030 } 2031 add_call_info(code_offset(), op->info()); 2032 } 2033 2034 2035 /* Currently, vtable-dispatch is only enabled for sparc platforms */ 2036 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) { 2037 ShouldNotReachHere(); 2038 } 2039 2040 2041 void LIR_Assembler::emit_static_call_stub() { 2042 address call_pc = __ pc(); 2043 address stub = __ start_a_stub(call_stub_size()); 2044 if (stub == NULL) { 2045 bailout("static call stub overflow"); 2046 return; 2047 } 2048 2049 int start = __ offset(); 2050 2051 __ relocate(static_stub_Relocation::spec(call_pc)); 2052 __ mov_metadata(rmethod, (Metadata*)NULL); 2053 __ movptr(rscratch1, 0); 2054 __ br(rscratch1); 2055 2056 assert(__ offset() - start <= call_stub_size(), "stub too big"); 2057 __ end_a_stub(); 2058 } 2059 2060 2061 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 2062 assert(exceptionOop->as_register() == r0, "must match"); 2063 assert(exceptionPC->as_register() == r3, "must match"); 2064 2065 // exception object is not added to oop map by LinearScan 2066 // (LinearScan assumes that no oops are in fixed registers) 2067 info->add_register_oop(exceptionOop); 2068 Runtime1::StubID unwind_id; 2069 2070 // get current pc information 2071 // pc is only needed if the method has an exception handler, the unwind code does not need it. 2072 int pc_for_athrow_offset = __ offset(); 2073 InternalAddress pc_for_athrow(__ pc()); 2074 __ adr(exceptionPC->as_register(), pc_for_athrow); 2075 add_call_info(pc_for_athrow_offset, info); // for exception handler 2076 2077 __ verify_not_null_oop(r0); 2078 // search an exception handler (r0: exception oop, r3: throwing pc) 2079 if (compilation()->has_fpu_code()) { 2080 unwind_id = Runtime1::handle_exception_id; 2081 } else { 2082 unwind_id = Runtime1::handle_exception_nofpu_id; 2083 } 2084 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 2085 2086 // FIXME: enough room for two byte trap ???? 2087 __ nop(); 2088 } 2089 2090 2091 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 2092 assert(exceptionOop->as_register() == r0, "must match"); 2093 2094 __ b(_unwind_handler_entry); 2095 } 2096 2097 2098 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 2099 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2100 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2101 2102 switch (left->type()) { 2103 case T_INT: { 2104 switch (code) { 2105 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break; 2106 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break; 2107 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break; 2108 default: 2109 ShouldNotReachHere(); 2110 break; 2111 } 2112 break; 2113 case T_LONG: 2114 case T_ADDRESS: 2115 case T_OBJECT: 2116 switch (code) { 2117 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break; 2118 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break; 2119 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break; 2120 default: 2121 ShouldNotReachHere(); 2122 break; 2123 } 2124 break; 2125 default: 2126 ShouldNotReachHere(); 2127 break; 2128 } 2129 } 2130 } 2131 2132 2133 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 2134 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2135 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2136 2137 switch (left->type()) { 2138 case T_INT: { 2139 switch (code) { 2140 case lir_shl: __ lslw (dreg, lreg, count); break; 2141 case lir_shr: __ asrw (dreg, lreg, count); break; 2142 case lir_ushr: __ lsrw (dreg, lreg, count); break; 2143 default: 2144 ShouldNotReachHere(); 2145 break; 2146 } 2147 break; 2148 case T_LONG: 2149 case T_ADDRESS: 2150 case T_OBJECT: 2151 switch (code) { 2152 case lir_shl: __ lsl (dreg, lreg, count); break; 2153 case lir_shr: __ asr (dreg, lreg, count); break; 2154 case lir_ushr: __ lsr (dreg, lreg, count); break; 2155 default: 2156 ShouldNotReachHere(); 2157 break; 2158 } 2159 break; 2160 default: 2161 ShouldNotReachHere(); 2162 break; 2163 } 2164 } 2165 } 2166 2167 2168 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 2169 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2170 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2171 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2172 __ str (r, Address(sp, offset_from_rsp_in_bytes)); 2173 } 2174 2175 2176 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 2177 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2178 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2179 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2180 __ mov (rscratch1, c); 2181 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2182 } 2183 2184 2185 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) { 2186 ShouldNotReachHere(); 2187 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2188 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2189 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2190 __ lea(rscratch1, __ constant_oop_address(o)); 2191 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2192 } 2193 2194 2195 // This code replaces a call to arraycopy; no exception may 2196 // be thrown in this code, they must be thrown in the System.arraycopy 2197 // activation frame; we could save some checks if this would not be the case 2198 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 2199 ciArrayKlass* default_type = op->expected_type(); 2200 Register src = op->src()->as_register(); 2201 Register dst = op->dst()->as_register(); 2202 Register src_pos = op->src_pos()->as_register(); 2203 Register dst_pos = op->dst_pos()->as_register(); 2204 Register length = op->length()->as_register(); 2205 Register tmp = op->tmp()->as_register(); 2206 2207 CodeStub* stub = op->stub(); 2208 int flags = op->flags(); 2209 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL; 2210 if (basic_type == T_ARRAY) basic_type = T_OBJECT; 2211 2212 // if we don't know anything, just go through the generic arraycopy 2213 if (default_type == NULL // || basic_type == T_OBJECT 2214 ) { 2215 Label done; 2216 assert(src == r1 && src_pos == r2, "mismatch in calling convention"); 2217 2218 // Save the arguments in case the generic arraycopy fails and we 2219 // have to fall back to the JNI stub 2220 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2221 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2222 __ str(src, Address(sp, 4*BytesPerWord)); 2223 2224 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy); 2225 address copyfunc_addr = StubRoutines::generic_arraycopy(); 2226 2227 // The arguments are in java calling convention so we shift them 2228 // to C convention 2229 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4); 2230 __ mov(c_rarg0, j_rarg0); 2231 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4); 2232 __ mov(c_rarg1, j_rarg1); 2233 assert_different_registers(c_rarg2, j_rarg3, j_rarg4); 2234 __ mov(c_rarg2, j_rarg2); 2235 assert_different_registers(c_rarg3, j_rarg4); 2236 __ mov(c_rarg3, j_rarg3); 2237 __ mov(c_rarg4, j_rarg4); 2238 if (copyfunc_addr == NULL) { // Use C version if stub was not generated 2239 __ mov(rscratch1, RuntimeAddress(C_entry)); 2240 __ blrt(rscratch1, 5, 0, 1); 2241 } else { 2242 #ifndef PRODUCT 2243 if (PrintC1Statistics) { 2244 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); 2245 } 2246 #endif 2247 __ far_call(RuntimeAddress(copyfunc_addr)); 2248 } 2249 2250 __ cbz(r0, *stub->continuation()); 2251 2252 // Reload values from the stack so they are where the stub 2253 // expects them. 2254 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2255 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2256 __ ldr(src, Address(sp, 4*BytesPerWord)); 2257 2258 if (copyfunc_addr != NULL) { 2259 // r0 is -1^K where K == partial copied count 2260 __ eonw(rscratch1, r0, 0); 2261 // adjust length down and src/end pos up by partial copied count 2262 __ subw(length, length, rscratch1); 2263 __ addw(src_pos, src_pos, rscratch1); 2264 __ addw(dst_pos, dst_pos, rscratch1); 2265 } 2266 __ b(*stub->entry()); 2267 2268 __ bind(*stub->continuation()); 2269 return; 2270 } 2271 2272 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point"); 2273 2274 int elem_size = type2aelembytes(basic_type); 2275 int shift_amount; 2276 int scale = exact_log2(elem_size); 2277 2278 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); 2279 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); 2280 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); 2281 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); 2282 2283 // test for NULL 2284 if (flags & LIR_OpArrayCopy::src_null_check) { 2285 __ cbz(src, *stub->entry()); 2286 } 2287 if (flags & LIR_OpArrayCopy::dst_null_check) { 2288 __ cbz(dst, *stub->entry()); 2289 } 2290 2291 // If the compiler was not able to prove that exact type of the source or the destination 2292 // of the arraycopy is an array type, check at runtime if the source or the destination is 2293 // an instance type. 2294 if (flags & LIR_OpArrayCopy::type_check) { 2295 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) { 2296 __ load_klass(tmp, dst); 2297 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2298 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2299 __ br(Assembler::GE, *stub->entry()); 2300 } 2301 2302 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) { 2303 __ load_klass(tmp, src); 2304 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2305 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2306 __ br(Assembler::GE, *stub->entry()); 2307 } 2308 } 2309 2310 // check if negative 2311 if (flags & LIR_OpArrayCopy::src_pos_positive_check) { 2312 __ cmpw(src_pos, 0); 2313 __ br(Assembler::LT, *stub->entry()); 2314 } 2315 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) { 2316 __ cmpw(dst_pos, 0); 2317 __ br(Assembler::LT, *stub->entry()); 2318 } 2319 2320 if (flags & LIR_OpArrayCopy::length_positive_check) { 2321 __ cmpw(length, 0); 2322 __ br(Assembler::LT, *stub->entry()); 2323 } 2324 2325 if (flags & LIR_OpArrayCopy::src_range_check) { 2326 __ addw(tmp, src_pos, length); 2327 __ ldrw(rscratch1, src_length_addr); 2328 __ cmpw(tmp, rscratch1); 2329 __ br(Assembler::HI, *stub->entry()); 2330 } 2331 if (flags & LIR_OpArrayCopy::dst_range_check) { 2332 __ addw(tmp, dst_pos, length); 2333 __ ldrw(rscratch1, dst_length_addr); 2334 __ cmpw(tmp, rscratch1); 2335 __ br(Assembler::HI, *stub->entry()); 2336 } 2337 2338 if (flags & LIR_OpArrayCopy::type_check) { 2339 // We don't know the array types are compatible 2340 if (basic_type != T_OBJECT) { 2341 // Simple test for basic type arrays 2342 if (UseCompressedClassPointers) { 2343 __ ldrw(tmp, src_klass_addr); 2344 __ ldrw(rscratch1, dst_klass_addr); 2345 __ cmpw(tmp, rscratch1); 2346 } else { 2347 __ ldr(tmp, src_klass_addr); 2348 __ ldr(rscratch1, dst_klass_addr); 2349 __ cmp(tmp, rscratch1); 2350 } 2351 __ br(Assembler::NE, *stub->entry()); 2352 } else { 2353 // For object arrays, if src is a sub class of dst then we can 2354 // safely do the copy. 2355 Label cont, slow; 2356 2357 #define PUSH(r1, r2) \ 2358 stp(r1, r2, __ pre(sp, -2 * wordSize)); 2359 2360 #define POP(r1, r2) \ 2361 ldp(r1, r2, __ post(sp, 2 * wordSize)); 2362 2363 __ PUSH(src, dst); 2364 2365 __ load_klass(src, src); 2366 __ load_klass(dst, dst); 2367 2368 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL); 2369 2370 __ PUSH(src, dst); 2371 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 2372 __ POP(src, dst); 2373 2374 __ cbnz(src, cont); 2375 2376 __ bind(slow); 2377 __ POP(src, dst); 2378 2379 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 2380 if (copyfunc_addr != NULL) { // use stub if available 2381 // src is not a sub class of dst so we have to do a 2382 // per-element check. 2383 2384 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 2385 if ((flags & mask) != mask) { 2386 // Check that at least both of them object arrays. 2387 assert(flags & mask, "one of the two should be known to be an object array"); 2388 2389 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 2390 __ load_klass(tmp, src); 2391 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 2392 __ load_klass(tmp, dst); 2393 } 2394 int lh_offset = in_bytes(Klass::layout_helper_offset()); 2395 Address klass_lh_addr(tmp, lh_offset); 2396 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 2397 __ ldrw(rscratch1, klass_lh_addr); 2398 __ mov(rscratch2, objArray_lh); 2399 __ eorw(rscratch1, rscratch1, rscratch2); 2400 __ cbnzw(rscratch1, *stub->entry()); 2401 } 2402 2403 // Spill because stubs can use any register they like and it's 2404 // easier to restore just those that we care about. 2405 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2406 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2407 __ str(src, Address(sp, 4*BytesPerWord)); 2408 2409 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2410 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2411 assert_different_registers(c_rarg0, dst, dst_pos, length); 2412 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2413 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2414 assert_different_registers(c_rarg1, dst, length); 2415 __ uxtw(c_rarg2, length); 2416 assert_different_registers(c_rarg2, dst); 2417 2418 __ load_klass(c_rarg4, dst); 2419 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset())); 2420 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset())); 2421 __ far_call(RuntimeAddress(copyfunc_addr)); 2422 2423 #ifndef PRODUCT 2424 if (PrintC1Statistics) { 2425 Label failed; 2426 __ cbnz(r0, failed); 2427 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt)); 2428 __ bind(failed); 2429 } 2430 #endif 2431 2432 __ cbz(r0, *stub->continuation()); 2433 2434 #ifndef PRODUCT 2435 if (PrintC1Statistics) { 2436 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt)); 2437 } 2438 #endif 2439 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1); 2440 2441 // Restore previously spilled arguments 2442 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2443 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2444 __ ldr(src, Address(sp, 4*BytesPerWord)); 2445 2446 // return value is -1^K where K is partial copied count 2447 __ eonw(rscratch1, r0, zr); 2448 // adjust length down and src/end pos up by partial copied count 2449 __ subw(length, length, rscratch1); 2450 __ addw(src_pos, src_pos, rscratch1); 2451 __ addw(dst_pos, dst_pos, rscratch1); 2452 } 2453 2454 __ b(*stub->entry()); 2455 2456 __ bind(cont); 2457 __ POP(src, dst); 2458 } 2459 } 2460 2461 #ifdef ASSERT 2462 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { 2463 // Sanity check the known type with the incoming class. For the 2464 // primitive case the types must match exactly with src.klass and 2465 // dst.klass each exactly matching the default type. For the 2466 // object array case, if no type check is needed then either the 2467 // dst type is exactly the expected type and the src type is a 2468 // subtype which we can't check or src is the same array as dst 2469 // but not necessarily exactly of type default_type. 2470 Label known_ok, halt; 2471 __ mov_metadata(tmp, default_type->constant_encoding()); 2472 if (UseCompressedClassPointers) { 2473 __ encode_klass_not_null(tmp); 2474 } 2475 2476 if (basic_type != T_OBJECT) { 2477 2478 if (UseCompressedClassPointers) { 2479 __ ldrw(rscratch1, dst_klass_addr); 2480 __ cmpw(tmp, rscratch1); 2481 } else { 2482 __ ldr(rscratch1, dst_klass_addr); 2483 __ cmp(tmp, rscratch1); 2484 } 2485 __ br(Assembler::NE, halt); 2486 if (UseCompressedClassPointers) { 2487 __ ldrw(rscratch1, src_klass_addr); 2488 __ cmpw(tmp, rscratch1); 2489 } else { 2490 __ ldr(rscratch1, src_klass_addr); 2491 __ cmp(tmp, rscratch1); 2492 } 2493 __ br(Assembler::EQ, known_ok); 2494 } else { 2495 if (UseCompressedClassPointers) { 2496 __ ldrw(rscratch1, dst_klass_addr); 2497 __ cmpw(tmp, rscratch1); 2498 } else { 2499 __ ldr(rscratch1, dst_klass_addr); 2500 __ cmp(tmp, rscratch1); 2501 } 2502 __ br(Assembler::EQ, known_ok); 2503 __ cmp(src, dst); 2504 __ br(Assembler::EQ, known_ok); 2505 } 2506 __ bind(halt); 2507 __ stop("incorrect type information in arraycopy"); 2508 __ bind(known_ok); 2509 } 2510 #endif 2511 2512 #ifndef PRODUCT 2513 if (PrintC1Statistics) { 2514 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type))); 2515 } 2516 #endif 2517 2518 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2519 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2520 assert_different_registers(c_rarg0, dst, dst_pos, length); 2521 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2522 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2523 assert_different_registers(c_rarg1, dst, length); 2524 __ uxtw(c_rarg2, length); 2525 assert_different_registers(c_rarg2, dst); 2526 2527 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 2528 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 2529 const char *name; 2530 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 2531 2532 CodeBlob *cb = CodeCache::find_blob(entry); 2533 if (cb) { 2534 __ far_call(RuntimeAddress(entry)); 2535 } else { 2536 __ call_VM_leaf(entry, 3); 2537 } 2538 2539 __ bind(*stub->continuation()); 2540 } 2541 2542 2543 2544 2545 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 2546 Register obj = op->obj_opr()->as_register(); // may not be an oop 2547 Register hdr = op->hdr_opr()->as_register(); 2548 Register lock = op->lock_opr()->as_register(); 2549 if (!UseFastLocking) { 2550 __ b(*op->stub()->entry()); 2551 } else if (op->code() == lir_lock) { 2552 Register scratch = noreg; 2553 if (UseBiasedLocking) { 2554 scratch = op->scratch_opr()->as_register(); 2555 } 2556 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2557 // add debug info for NullPointerException only if one is possible 2558 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry()); 2559 if (op->info() != NULL) { 2560 add_debug_info_for_null_check(null_check_offset, op->info()); 2561 } 2562 // done 2563 } else if (op->code() == lir_unlock) { 2564 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2565 __ unlock_object(hdr, obj, lock, *op->stub()->entry()); 2566 } else { 2567 Unimplemented(); 2568 } 2569 __ bind(*op->stub()->continuation()); 2570 } 2571 2572 2573 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 2574 ciMethod* method = op->profiled_method(); 2575 int bci = op->profiled_bci(); 2576 ciMethod* callee = op->profiled_callee(); 2577 2578 // Update counter for all call types 2579 ciMethodData* md = method->method_data_or_null(); 2580 assert(md != NULL, "Sanity"); 2581 ciProfileData* data = md->bci_to_data(bci); 2582 assert(data->is_CounterData(), "need CounterData for calls"); 2583 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 2584 Register mdo = op->mdo()->as_register(); 2585 __ mov_metadata(mdo, md->constant_encoding()); 2586 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 2587 Bytecodes::Code bc = method->java_code_at_bci(bci); 2588 const bool callee_is_static = callee->is_loaded() && callee->is_static(); 2589 // Perform additional virtual call profiling for invokevirtual and 2590 // invokeinterface bytecodes 2591 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) && 2592 !callee_is_static && // required for optimized MH invokes 2593 C1ProfileVirtualCalls) { 2594 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 2595 Register recv = op->recv()->as_register(); 2596 assert_different_registers(mdo, recv); 2597 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 2598 ciKlass* known_klass = op->known_holder(); 2599 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) { 2600 // We know the type that will be seen at this call site; we can 2601 // statically update the MethodData* rather than needing to do 2602 // dynamic tests on the receiver type 2603 2604 // NOTE: we should probably put a lock around this search to 2605 // avoid collisions by concurrent compilations 2606 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 2607 uint i; 2608 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2609 ciKlass* receiver = vc_data->receiver(i); 2610 if (known_klass->equals(receiver)) { 2611 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2612 __ addptr(data_addr, DataLayout::counter_increment); 2613 return; 2614 } 2615 } 2616 2617 // Receiver type not found in profile data; select an empty slot 2618 2619 // Note that this is less efficient than it should be because it 2620 // always does a write to the receiver part of the 2621 // VirtualCallData rather than just the first time 2622 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2623 ciKlass* receiver = vc_data->receiver(i); 2624 if (receiver == NULL) { 2625 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); 2626 __ mov_metadata(rscratch1, known_klass->constant_encoding()); 2627 __ lea(rscratch2, recv_addr); 2628 __ str(rscratch1, Address(rscratch2)); 2629 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2630 __ addptr(data_addr, DataLayout::counter_increment); 2631 return; 2632 } 2633 } 2634 } else { 2635 __ load_klass(recv, recv); 2636 Label update_done; 2637 type_profile_helper(mdo, md, data, recv, &update_done); 2638 // Receiver did not match any saved receiver and there is no empty row for it. 2639 // Increment total counter to indicate polymorphic case. 2640 __ addptr(counter_addr, DataLayout::counter_increment); 2641 2642 __ bind(update_done); 2643 } 2644 } else { 2645 // Static call 2646 __ addptr(counter_addr, DataLayout::counter_increment); 2647 } 2648 } 2649 2650 2651 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 2652 Unimplemented(); 2653 } 2654 2655 2656 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 2657 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 2658 } 2659 2660 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 2661 assert(op->crc()->is_single_cpu(), "crc must be register"); 2662 assert(op->val()->is_single_cpu(), "byte value must be register"); 2663 assert(op->result_opr()->is_single_cpu(), "result must be register"); 2664 Register crc = op->crc()->as_register(); 2665 Register val = op->val()->as_register(); 2666 Register res = op->result_opr()->as_register(); 2667 2668 assert_different_registers(val, crc, res); 2669 unsigned long offset; 2670 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset); 2671 if (offset) __ add(res, res, offset); 2672 2673 __ ornw(crc, zr, crc); // ~crc 2674 __ update_byte_crc32(crc, val, res); 2675 __ ornw(res, zr, crc); // ~crc 2676 } 2677 2678 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 2679 COMMENT("emit_profile_type {"); 2680 Register obj = op->obj()->as_register(); 2681 Register tmp = op->tmp()->as_pointer_register(); 2682 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 2683 ciKlass* exact_klass = op->exact_klass(); 2684 intptr_t current_klass = op->current_klass(); 2685 bool not_null = op->not_null(); 2686 bool no_conflict = op->no_conflict(); 2687 2688 Label update, next, none; 2689 2690 bool do_null = !not_null; 2691 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 2692 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 2693 2694 assert(do_null || do_update, "why are we here?"); 2695 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 2696 assert(mdo_addr.base() != rscratch1, "wrong register"); 2697 2698 __ verify_oop(obj); 2699 2700 if (tmp != obj) { 2701 __ mov(tmp, obj); 2702 } 2703 if (do_null) { 2704 __ cbnz(tmp, update); 2705 if (!TypeEntries::was_null_seen(current_klass)) { 2706 __ ldr(rscratch2, mdo_addr); 2707 __ orr(rscratch2, rscratch2, TypeEntries::null_seen); 2708 __ str(rscratch2, mdo_addr); 2709 } 2710 if (do_update) { 2711 #ifndef ASSERT 2712 __ b(next); 2713 } 2714 #else 2715 __ b(next); 2716 } 2717 } else { 2718 __ cbnz(tmp, update); 2719 __ stop("unexpected null obj"); 2720 #endif 2721 } 2722 2723 __ bind(update); 2724 2725 if (do_update) { 2726 #ifdef ASSERT 2727 if (exact_klass != NULL) { 2728 Label ok; 2729 __ load_klass(tmp, tmp); 2730 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2731 __ eor(rscratch1, tmp, rscratch1); 2732 __ cbz(rscratch1, ok); 2733 __ stop("exact klass and actual klass differ"); 2734 __ bind(ok); 2735 } 2736 #endif 2737 if (!no_conflict) { 2738 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) { 2739 if (exact_klass != NULL) { 2740 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2741 } else { 2742 __ load_klass(tmp, tmp); 2743 } 2744 2745 __ ldr(rscratch2, mdo_addr); 2746 __ eor(tmp, tmp, rscratch2); 2747 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2748 // klass seen before, nothing to do. The unknown bit may have been 2749 // set already but no need to check. 2750 __ cbz(rscratch1, next); 2751 2752 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2753 2754 if (TypeEntries::is_type_none(current_klass)) { 2755 __ cbz(rscratch2, none); 2756 __ cmp(rscratch2, TypeEntries::null_seen); 2757 __ br(Assembler::EQ, none); 2758 // There is a chance that the checks above (re-reading profiling 2759 // data from memory) fail if another thread has just set the 2760 // profiling to this obj's klass 2761 __ dmb(Assembler::ISHLD); 2762 __ ldr(rscratch2, mdo_addr); 2763 __ eor(tmp, tmp, rscratch2); 2764 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2765 __ cbz(rscratch1, next); 2766 } 2767 } else { 2768 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 2769 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 2770 2771 __ ldr(tmp, mdo_addr); 2772 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2773 } 2774 2775 // different than before. Cannot keep accurate profile. 2776 __ ldr(rscratch2, mdo_addr); 2777 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown); 2778 __ str(rscratch2, mdo_addr); 2779 2780 if (TypeEntries::is_type_none(current_klass)) { 2781 __ b(next); 2782 2783 __ bind(none); 2784 // first time here. Set profile type. 2785 __ str(tmp, mdo_addr); 2786 } 2787 } else { 2788 // There's a single possible klass at this profile point 2789 assert(exact_klass != NULL, "should be"); 2790 if (TypeEntries::is_type_none(current_klass)) { 2791 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2792 __ ldr(rscratch2, mdo_addr); 2793 __ eor(tmp, tmp, rscratch2); 2794 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2795 __ cbz(rscratch1, next); 2796 #ifdef ASSERT 2797 { 2798 Label ok; 2799 __ ldr(rscratch1, mdo_addr); 2800 __ cbz(rscratch1, ok); 2801 __ cmp(rscratch1, TypeEntries::null_seen); 2802 __ br(Assembler::EQ, ok); 2803 // may have been set by another thread 2804 __ dmb(Assembler::ISHLD); 2805 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2806 __ ldr(rscratch2, mdo_addr); 2807 __ eor(rscratch2, rscratch1, rscratch2); 2808 __ andr(rscratch2, rscratch2, TypeEntries::type_mask); 2809 __ cbz(rscratch2, ok); 2810 2811 __ stop("unexpected profiling mismatch"); 2812 __ bind(ok); 2813 } 2814 #endif 2815 // first time here. Set profile type. 2816 __ ldr(tmp, mdo_addr); 2817 } else { 2818 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 2819 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 2820 2821 __ ldr(tmp, mdo_addr); 2822 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2823 2824 __ orr(tmp, tmp, TypeEntries::type_unknown); 2825 __ str(tmp, mdo_addr); 2826 // FIXME: Write barrier needed here? 2827 } 2828 } 2829 2830 __ bind(next); 2831 } 2832 COMMENT("} emit_profile_type"); 2833 } 2834 2835 2836 void LIR_Assembler::align_backward_branch_target() { 2837 } 2838 2839 2840 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) { 2841 if (left->is_single_cpu()) { 2842 assert(dest->is_single_cpu(), "expect single result reg"); 2843 __ negw(dest->as_register(), left->as_register()); 2844 } else if (left->is_double_cpu()) { 2845 assert(dest->is_double_cpu(), "expect double result reg"); 2846 __ neg(dest->as_register_lo(), left->as_register_lo()); 2847 } else if (left->is_single_fpu()) { 2848 assert(dest->is_single_fpu(), "expect single float result reg"); 2849 __ fnegs(dest->as_float_reg(), left->as_float_reg()); 2850 } else { 2851 assert(left->is_double_fpu(), "expect double float operand reg"); 2852 assert(dest->is_double_fpu(), "expect double float result reg"); 2853 __ fnegd(dest->as_double_reg(), left->as_double_reg()); 2854 } 2855 } 2856 2857 2858 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) { 2859 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr())); 2860 } 2861 2862 2863 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 2864 assert(!tmp->is_valid(), "don't need temporary"); 2865 2866 CodeBlob *cb = CodeCache::find_blob(dest); 2867 if (cb) { 2868 __ far_call(RuntimeAddress(dest)); 2869 } else { 2870 __ mov(rscratch1, RuntimeAddress(dest)); 2871 int len = args->length(); 2872 int type = 0; 2873 if (! result->is_illegal()) { 2874 switch (result->type()) { 2875 case T_VOID: 2876 type = 0; 2877 break; 2878 case T_INT: 2879 case T_LONG: 2880 case T_OBJECT: 2881 type = 1; 2882 break; 2883 case T_FLOAT: 2884 type = 2; 2885 break; 2886 case T_DOUBLE: 2887 type = 3; 2888 break; 2889 default: 2890 ShouldNotReachHere(); 2891 break; 2892 } 2893 } 2894 int num_gpargs = 0; 2895 int num_fpargs = 0; 2896 for (int i = 0; i < args->length(); i++) { 2897 LIR_Opr arg = args->at(i); 2898 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) { 2899 num_fpargs++; 2900 } else { 2901 num_gpargs++; 2902 } 2903 } 2904 __ blrt(rscratch1, num_gpargs, num_fpargs, type); 2905 } 2906 2907 if (info != NULL) { 2908 add_call_info_here(info); 2909 } 2910 __ maybe_isb(); 2911 } 2912 2913 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 2914 if (dest->is_address() || src->is_address()) { 2915 move_op(src, dest, type, lir_patch_none, info, 2916 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false); 2917 } else { 2918 ShouldNotReachHere(); 2919 } 2920 } 2921 2922 #ifdef ASSERT 2923 // emit run-time assertion 2924 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 2925 assert(op->code() == lir_assert, "must be"); 2926 2927 if (op->in_opr1()->is_valid()) { 2928 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 2929 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 2930 } else { 2931 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 2932 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 2933 } 2934 2935 Label ok; 2936 if (op->condition() != lir_cond_always) { 2937 Assembler::Condition acond = Assembler::AL; 2938 switch (op->condition()) { 2939 case lir_cond_equal: acond = Assembler::EQ; break; 2940 case lir_cond_notEqual: acond = Assembler::NE; break; 2941 case lir_cond_less: acond = Assembler::LT; break; 2942 case lir_cond_lessEqual: acond = Assembler::LE; break; 2943 case lir_cond_greaterEqual: acond = Assembler::GE; break; 2944 case lir_cond_greater: acond = Assembler::GT; break; 2945 case lir_cond_belowEqual: acond = Assembler::LS; break; 2946 case lir_cond_aboveEqual: acond = Assembler::HS; break; 2947 default: ShouldNotReachHere(); 2948 } 2949 __ br(acond, ok); 2950 } 2951 if (op->halt()) { 2952 const char* str = __ code_string(op->msg()); 2953 __ stop(str); 2954 } else { 2955 breakpoint(); 2956 } 2957 __ bind(ok); 2958 } 2959 #endif 2960 2961 #ifndef PRODUCT 2962 #define COMMENT(x) do { __ block_comment(x); } while (0) 2963 #else 2964 #define COMMENT(x) 2965 #endif 2966 2967 void LIR_Assembler::membar() { 2968 COMMENT("membar"); 2969 __ membar(MacroAssembler::AnyAny); 2970 } 2971 2972 void LIR_Assembler::membar_acquire() { 2973 __ membar(Assembler::LoadLoad|Assembler::LoadStore); 2974 } 2975 2976 void LIR_Assembler::membar_release() { 2977 __ membar(Assembler::LoadStore|Assembler::StoreStore); 2978 } 2979 2980 void LIR_Assembler::membar_loadload() { 2981 __ membar(Assembler::LoadLoad); 2982 } 2983 2984 void LIR_Assembler::membar_storestore() { 2985 __ membar(MacroAssembler::StoreStore); 2986 } 2987 2988 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); } 2989 2990 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); } 2991 2992 void LIR_Assembler::on_spin_wait() { 2993 Unimplemented(); 2994 } 2995 2996 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 2997 __ mov(result_reg->as_register(), rthread); 2998 } 2999 3000 3001 void LIR_Assembler::peephole(LIR_List *lir) { 3002 #if 0 3003 if (tableswitch_count >= max_tableswitches) 3004 return; 3005 3006 /* 3007 This finite-state automaton recognizes sequences of compare-and- 3008 branch instructions. We will turn them into a tableswitch. You 3009 could argue that C1 really shouldn't be doing this sort of 3010 optimization, but without it the code is really horrible. 3011 */ 3012 3013 enum { start_s, cmp1_s, beq_s, cmp_s } state; 3014 int first_key, last_key = -2147483648; 3015 int next_key = 0; 3016 int start_insn = -1; 3017 int last_insn = -1; 3018 Register reg = noreg; 3019 LIR_Opr reg_opr; 3020 state = start_s; 3021 3022 LIR_OpList* inst = lir->instructions_list(); 3023 for (int i = 0; i < inst->length(); i++) { 3024 LIR_Op* op = inst->at(i); 3025 switch (state) { 3026 case start_s: 3027 first_key = -1; 3028 start_insn = i; 3029 switch (op->code()) { 3030 case lir_cmp: 3031 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3032 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3033 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3034 && opr2->is_constant() 3035 && opr2->type() == T_INT) { 3036 reg_opr = opr1; 3037 reg = opr1->as_register(); 3038 first_key = opr2->as_constant_ptr()->as_jint(); 3039 next_key = first_key + 1; 3040 state = cmp_s; 3041 goto next_state; 3042 } 3043 break; 3044 } 3045 break; 3046 case cmp_s: 3047 switch (op->code()) { 3048 case lir_branch: 3049 if (op->as_OpBranch()->cond() == lir_cond_equal) { 3050 state = beq_s; 3051 last_insn = i; 3052 goto next_state; 3053 } 3054 } 3055 state = start_s; 3056 break; 3057 case beq_s: 3058 switch (op->code()) { 3059 case lir_cmp: { 3060 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3061 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3062 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3063 && opr1->as_register() == reg 3064 && opr2->is_constant() 3065 && opr2->type() == T_INT 3066 && opr2->as_constant_ptr()->as_jint() == next_key) { 3067 last_key = next_key; 3068 next_key++; 3069 state = cmp_s; 3070 goto next_state; 3071 } 3072 } 3073 } 3074 last_key = next_key; 3075 state = start_s; 3076 break; 3077 default: 3078 assert(false, "impossible state"); 3079 } 3080 if (state == start_s) { 3081 if (first_key < last_key - 5L && reg != noreg) { 3082 { 3083 // printf("found run register %d starting at insn %d low value %d high value %d\n", 3084 // reg->encoding(), 3085 // start_insn, first_key, last_key); 3086 // for (int i = 0; i < inst->length(); i++) { 3087 // inst->at(i)->print(); 3088 // tty->print("\n"); 3089 // } 3090 // tty->print("\n"); 3091 } 3092 3093 struct tableswitch *sw = &switches[tableswitch_count]; 3094 sw->_insn_index = start_insn, sw->_first_key = first_key, 3095 sw->_last_key = last_key, sw->_reg = reg; 3096 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after)); 3097 { 3098 // Insert the new table of branches 3099 int offset = last_insn; 3100 for (int n = first_key; n < last_key; n++) { 3101 inst->insert_before 3102 (last_insn + 1, 3103 new LIR_OpBranch(lir_cond_always, T_ILLEGAL, 3104 inst->at(offset)->as_OpBranch()->label())); 3105 offset -= 2, i++; 3106 } 3107 } 3108 // Delete all the old compare-and-branch instructions 3109 for (int n = first_key; n < last_key; n++) { 3110 inst->remove_at(start_insn); 3111 inst->remove_at(start_insn); 3112 } 3113 // Insert the tableswitch instruction 3114 inst->insert_before(start_insn, 3115 new LIR_Op2(lir_cmp, lir_cond_always, 3116 LIR_OprFact::intConst(tableswitch_count), 3117 reg_opr)); 3118 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches)); 3119 tableswitch_count++; 3120 } 3121 reg = noreg; 3122 last_key = -2147483648; 3123 } 3124 next_state: 3125 ; 3126 } 3127 #endif 3128 } 3129 3130 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { 3131 Address addr = as_Address(src->as_address_ptr()); 3132 BasicType type = src->type(); 3133 bool is_oop = type == T_OBJECT || type == T_ARRAY; 3134 3135 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr); 3136 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr); 3137 3138 switch(type) { 3139 case T_INT: 3140 xchg = &MacroAssembler::atomic_xchgalw; 3141 add = &MacroAssembler::atomic_addalw; 3142 break; 3143 case T_LONG: 3144 xchg = &MacroAssembler::atomic_xchgal; 3145 add = &MacroAssembler::atomic_addal; 3146 break; 3147 case T_OBJECT: 3148 case T_ARRAY: 3149 if (UseCompressedOops) { 3150 xchg = &MacroAssembler::atomic_xchgalw; 3151 add = &MacroAssembler::atomic_addalw; 3152 } else { 3153 xchg = &MacroAssembler::atomic_xchgal; 3154 add = &MacroAssembler::atomic_addal; 3155 } 3156 break; 3157 default: 3158 ShouldNotReachHere(); 3159 xchg = &MacroAssembler::atomic_xchgal; 3160 add = &MacroAssembler::atomic_addal; // unreachable 3161 } 3162 3163 switch (code) { 3164 case lir_xadd: 3165 { 3166 RegisterOrConstant inc; 3167 Register tmp = as_reg(tmp_op); 3168 Register dst = as_reg(dest); 3169 if (data->is_constant()) { 3170 inc = RegisterOrConstant(as_long(data)); 3171 assert_different_registers(dst, addr.base(), tmp, 3172 rscratch1, rscratch2); 3173 } else { 3174 inc = RegisterOrConstant(as_reg(data)); 3175 assert_different_registers(inc.as_register(), dst, addr.base(), tmp, 3176 rscratch1, rscratch2); 3177 } 3178 __ lea(tmp, addr); 3179 (_masm->*add)(dst, inc, tmp); 3180 break; 3181 } 3182 case lir_xchg: 3183 { 3184 Register tmp = tmp_op->as_register(); 3185 Register obj = as_reg(data); 3186 Register dst = as_reg(dest); 3187 if (is_oop && UseCompressedOops) { 3188 __ encode_heap_oop(rscratch2, obj); 3189 obj = rscratch2; 3190 } 3191 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst); 3192 __ lea(tmp, addr); 3193 (_masm->*xchg)(dst, obj, tmp); 3194 if (is_oop && UseCompressedOops) { 3195 __ decode_heap_oop(dst); 3196 } 3197 } 3198 break; 3199 default: 3200 ShouldNotReachHere(); 3201 } 3202 __ membar(__ AnyAny); 3203 } 3204 3205 #undef __