1 /* 2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, Red Hat Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "asm/assembler.hpp" 28 #include "c1/c1_CodeStubs.hpp" 29 #include "c1/c1_Compilation.hpp" 30 #include "c1/c1_LIRAssembler.hpp" 31 #include "c1/c1_MacroAssembler.hpp" 32 #include "c1/c1_Runtime1.hpp" 33 #include "c1/c1_ValueStack.hpp" 34 #include "ci/ciArrayKlass.hpp" 35 #include "ci/ciInstance.hpp" 36 #include "gc/shared/barrierSet.hpp" 37 #include "gc/shared/cardTableModRefBS.hpp" 38 #include "gc/shared/collectedHeap.hpp" 39 #include "nativeInst_aarch64.hpp" 40 #include "oops/objArrayKlass.hpp" 41 #include "runtime/sharedRuntime.hpp" 42 #include "vmreg_aarch64.inline.hpp" 43 44 45 46 #ifndef PRODUCT 47 #define COMMENT(x) do { __ block_comment(x); } while (0) 48 #else 49 #define COMMENT(x) 50 #endif 51 52 NEEDS_CLEANUP // remove this definitions ? 53 const Register IC_Klass = rscratch2; // where the IC klass is cached 54 const Register SYNC_header = r0; // synchronization header 55 const Register SHIFT_count = r0; // where count for shift operations must be 56 57 #define __ _masm-> 58 59 60 static void select_different_registers(Register preserve, 61 Register extra, 62 Register &tmp1, 63 Register &tmp2) { 64 if (tmp1 == preserve) { 65 assert_different_registers(tmp1, tmp2, extra); 66 tmp1 = extra; 67 } else if (tmp2 == preserve) { 68 assert_different_registers(tmp1, tmp2, extra); 69 tmp2 = extra; 70 } 71 assert_different_registers(preserve, tmp1, tmp2); 72 } 73 74 75 76 static void select_different_registers(Register preserve, 77 Register extra, 78 Register &tmp1, 79 Register &tmp2, 80 Register &tmp3) { 81 if (tmp1 == preserve) { 82 assert_different_registers(tmp1, tmp2, tmp3, extra); 83 tmp1 = extra; 84 } else if (tmp2 == preserve) { 85 assert_different_registers(tmp1, tmp2, tmp3, extra); 86 tmp2 = extra; 87 } else if (tmp3 == preserve) { 88 assert_different_registers(tmp1, tmp2, tmp3, extra); 89 tmp3 = extra; 90 } 91 assert_different_registers(preserve, tmp1, tmp2, tmp3); 92 } 93 94 95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } 96 97 98 LIR_Opr LIR_Assembler::receiverOpr() { 99 return FrameMap::receiver_opr; 100 } 101 102 LIR_Opr LIR_Assembler::osrBufferPointer() { 103 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 104 } 105 106 //--------------fpu register translations----------------------- 107 108 109 address LIR_Assembler::float_constant(float f) { 110 address const_addr = __ float_constant(f); 111 if (const_addr == NULL) { 112 bailout("const section overflow"); 113 return __ code()->consts()->start(); 114 } else { 115 return const_addr; 116 } 117 } 118 119 120 address LIR_Assembler::double_constant(double d) { 121 address const_addr = __ double_constant(d); 122 if (const_addr == NULL) { 123 bailout("const section overflow"); 124 return __ code()->consts()->start(); 125 } else { 126 return const_addr; 127 } 128 } 129 130 address LIR_Assembler::int_constant(jlong n) { 131 address const_addr = __ long_constant(n); 132 if (const_addr == NULL) { 133 bailout("const section overflow"); 134 return __ code()->consts()->start(); 135 } else { 136 return const_addr; 137 } 138 } 139 140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); } 141 142 void LIR_Assembler::reset_FPU() { Unimplemented(); } 143 144 void LIR_Assembler::fpop() { Unimplemented(); } 145 146 void LIR_Assembler::fxch(int i) { Unimplemented(); } 147 148 void LIR_Assembler::fld(int i) { Unimplemented(); } 149 150 void LIR_Assembler::ffree(int i) { Unimplemented(); } 151 152 void LIR_Assembler::breakpoint() { Unimplemented(); } 153 154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); } 155 156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); } 157 158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; } 159 //------------------------------------------- 160 161 static Register as_reg(LIR_Opr op) { 162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register(); 163 } 164 165 static jlong as_long(LIR_Opr data) { 166 jlong result; 167 switch (data->type()) { 168 case T_INT: 169 result = (data->as_jint()); 170 break; 171 case T_LONG: 172 result = (data->as_jlong()); 173 break; 174 default: 175 ShouldNotReachHere(); 176 result = 0; // unreachable 177 } 178 return result; 179 } 180 181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 182 Register base = addr->base()->as_pointer_register(); 183 LIR_Opr opr = addr->index(); 184 if (opr->is_cpu_register()) { 185 Register index; 186 if (opr->is_single_cpu()) 187 index = opr->as_register(); 188 else 189 index = opr->as_register_lo(); 190 assert(addr->disp() == 0, "must be"); 191 switch(opr->type()) { 192 case T_INT: 193 return Address(base, index, Address::sxtw(addr->scale())); 194 case T_LONG: 195 return Address(base, index, Address::lsl(addr->scale())); 196 default: 197 ShouldNotReachHere(); 198 } 199 } else { 200 intptr_t addr_offset = intptr_t(addr->disp()); 201 if (Address::offset_ok_for_immed(addr_offset, addr->scale())) 202 return Address(base, addr_offset, Address::lsl(addr->scale())); 203 else { 204 __ mov(tmp, addr_offset); 205 return Address(base, tmp, Address::lsl(addr->scale())); 206 } 207 } 208 return Address(); 209 } 210 211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 212 ShouldNotReachHere(); 213 return Address(); 214 } 215 216 Address LIR_Assembler::as_Address(LIR_Address* addr) { 217 return as_Address(addr, rscratch1); 218 } 219 220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 221 return as_Address(addr, rscratch1); // Ouch 222 // FIXME: This needs to be much more clever. See x86. 223 } 224 225 226 void LIR_Assembler::osr_entry() { 227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 228 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 229 ValueStack* entry_state = osr_entry->state(); 230 int number_of_locks = entry_state->locks_size(); 231 232 // we jump here if osr happens with the interpreter 233 // state set up to continue at the beginning of the 234 // loop that triggered osr - in particular, we have 235 // the following registers setup: 236 // 237 // r2: osr buffer 238 // 239 240 // build frame 241 ciMethod* m = compilation()->method(); 242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 243 244 // OSR buffer is 245 // 246 // locals[nlocals-1..0] 247 // monitors[0..number_of_locks] 248 // 249 // locals is a direct copy of the interpreter frame so in the osr buffer 250 // so first slot in the local array is the last local from the interpreter 251 // and last slot is local[0] (receiver) from the interpreter 252 // 253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 255 // in the interpreter frame (the method lock if a sync method) 256 257 // Initialize monitors in the compiled activation. 258 // r2: pointer to osr buffer 259 // 260 // All other registers are dead at this point and the locals will be 261 // copied into place by code emitted in the IR. 262 263 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 265 int monitor_offset = BytesPerWord * method()->max_locals() + 266 (2 * BytesPerWord) * (number_of_locks - 1); 267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 268 // the OSR buffer using 2 word entries: first the lock and then 269 // the oop. 270 for (int i = 0; i < number_of_locks; i++) { 271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 272 #ifdef ASSERT 273 // verify the interpreter's monitor has a non-null object 274 { 275 Label L; 276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 277 __ cbnz(rscratch1, L); 278 __ stop("locked object is NULL"); 279 __ bind(L); 280 } 281 #endif 282 __ ldr(r19, Address(OSR_buf, slot_offset + 0)); 283 __ str(r19, frame_map()->address_for_monitor_lock(i)); 284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 285 __ str(r19, frame_map()->address_for_monitor_object(i)); 286 } 287 } 288 } 289 290 291 // inline cache check; done before the frame is built. 292 int LIR_Assembler::check_icache() { 293 Register receiver = FrameMap::receiver_opr->as_register(); 294 Register ic_klass = IC_Klass; 295 int start_offset = __ offset(); 296 __ inline_cache_check(receiver, ic_klass); 297 298 // if icache check fails, then jump to runtime routine 299 // Note: RECEIVER must still contain the receiver! 300 Label dont; 301 __ br(Assembler::EQ, dont); 302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 303 304 // We align the verified entry point unless the method body 305 // (including its inline cache check) will fit in a single 64-byte 306 // icache line. 307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) { 308 // force alignment after the cache check. 309 __ align(CodeEntryAlignment); 310 } 311 312 __ bind(dont); 313 return start_offset; 314 } 315 316 317 void LIR_Assembler::jobject2reg(jobject o, Register reg) { 318 if (o == NULL) { 319 __ mov(reg, zr); 320 } else { 321 __ movoop(reg, o, /*immediate*/true); 322 } 323 } 324 325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) { 326 address target = NULL; 327 relocInfo::relocType reloc_type = relocInfo::none; 328 329 switch (patching_id(info)) { 330 case PatchingStub::access_field_id: 331 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 332 reloc_type = relocInfo::section_word_type; 333 break; 334 case PatchingStub::load_klass_id: 335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 336 reloc_type = relocInfo::metadata_type; 337 break; 338 case PatchingStub::load_mirror_id: 339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 340 reloc_type = relocInfo::oop_type; 341 break; 342 case PatchingStub::load_appendix_id: 343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 344 reloc_type = relocInfo::oop_type; 345 break; 346 default: ShouldNotReachHere(); 347 } 348 349 __ far_call(RuntimeAddress(target)); 350 add_call_info_here(info); 351 } 352 353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) { 354 deoptimize_trap(info); 355 } 356 357 358 // This specifies the rsp decrement needed to build the frame 359 int LIR_Assembler::initial_frame_size_in_bytes() const { 360 // if rounding, must let FrameMap know! 361 362 // The frame_map records size in slots (32bit word) 363 364 // subtract two words to account for return address and link 365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size; 366 } 367 368 369 int LIR_Assembler::emit_exception_handler() { 370 // if the last instruction is a call (typically to do a throw which 371 // is coming at the end after block reordering) the return address 372 // must still point into the code area in order to avoid assertion 373 // failures when searching for the corresponding bci => add a nop 374 // (was bug 5/14/1999 - gri) 375 __ nop(); 376 377 // generate code for exception handler 378 address handler_base = __ start_a_stub(exception_handler_size); 379 if (handler_base == NULL) { 380 // not enough space left for the handler 381 bailout("exception handler overflow"); 382 return -1; 383 } 384 385 int offset = code_offset(); 386 387 // the exception oop and pc are in r0, and r3 388 // no other registers need to be preserved, so invalidate them 389 __ invalidate_registers(false, true, true, false, true, true); 390 391 // check that there is really an exception 392 __ verify_not_null_oop(r0); 393 394 // search an exception handler (r0: exception oop, r3: throwing pc) 395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here(); 396 guarantee(code_offset() - offset <= exception_handler_size, "overflow"); 397 __ end_a_stub(); 398 399 return offset; 400 } 401 402 403 // Emit the code to remove the frame from the stack in the exception 404 // unwind path. 405 int LIR_Assembler::emit_unwind_handler() { 406 #ifndef PRODUCT 407 if (CommentedAssembly) { 408 _masm->block_comment("Unwind handler"); 409 } 410 #endif 411 412 int offset = code_offset(); 413 414 // Fetch the exception from TLS and clear out exception related thread state 415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset())); 416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset())); 417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset())); 418 419 __ bind(_unwind_handler_entry); 420 __ verify_not_null_oop(r0); 421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 422 __ mov(r19, r0); // Preserve the exception 423 } 424 425 // Preform needed unlocking 426 MonitorExitStub* stub = NULL; 427 if (method()->is_synchronized()) { 428 monitor_address(0, FrameMap::r0_opr); 429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0); 430 __ unlock_object(r5, r4, r0, *stub->entry()); 431 __ bind(*stub->continuation()); 432 } 433 434 if (compilation()->env()->dtrace_method_probes()) { 435 __ call_Unimplemented(); 436 #if 0 437 __ movptr(Address(rsp, 0), rax); 438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding()); 439 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit))); 440 #endif 441 } 442 443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 444 __ mov(r0, r19); // Restore the exception 445 } 446 447 // remove the activation and dispatch to the unwind handler 448 __ block_comment("remove_frame and dispatch to the unwind handler"); 449 __ remove_frame(initial_frame_size_in_bytes()); 450 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id))); 451 452 // Emit the slow path assembly 453 if (stub != NULL) { 454 stub->emit_code(this); 455 } 456 457 return offset; 458 } 459 460 461 int LIR_Assembler::emit_deopt_handler() { 462 // if the last instruction is a call (typically to do a throw which 463 // is coming at the end after block reordering) the return address 464 // must still point into the code area in order to avoid assertion 465 // failures when searching for the corresponding bci => add a nop 466 // (was bug 5/14/1999 - gri) 467 __ nop(); 468 469 // generate code for exception handler 470 address handler_base = __ start_a_stub(deopt_handler_size); 471 if (handler_base == NULL) { 472 // not enough space left for the handler 473 bailout("deopt handler overflow"); 474 return -1; 475 } 476 477 int offset = code_offset(); 478 479 __ adr(lr, pc()); 480 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 481 guarantee(code_offset() - offset <= deopt_handler_size, "overflow"); 482 __ end_a_stub(); 483 484 return offset; 485 } 486 487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) { 488 _masm->code_section()->relocate(adr, relocInfo::poll_type); 489 int pc_offset = code_offset(); 490 flush_debug_info(pc_offset); 491 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset); 492 if (info->exception_handlers() != NULL) { 493 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers()); 494 } 495 } 496 497 // Rather than take a segfault when the polling page is protected, 498 // explicitly check for a safepoint in progress and if there is one, 499 // fake a call to the handler as if a segfault had been caught. 500 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) { 501 __ mov(rscratch1, SafepointSynchronize::address_of_state()); 502 __ ldrb(rscratch1, Address(rscratch1)); 503 Label nope, poll; 504 __ cbz(rscratch1, nope); 505 __ block_comment("safepoint"); 506 __ enter(); 507 __ push(0x3, sp); // r0 & r1 508 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1 509 __ adr(r0, poll); 510 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset())); 511 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub)); 512 __ blrt(rscratch1, 1, 0, 1); 513 __ maybe_isb(); 514 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1 515 __ mov(rscratch1, r0); 516 __ pop(0x3, sp); // r0 & r1 517 __ leave(); 518 __ br(rscratch1); 519 address polling_page(os::get_polling_page()); 520 assert(os::is_poll_address(polling_page), "should be"); 521 unsigned long off; 522 __ adrp(rscratch1, Address(polling_page, rtype), off); 523 __ bind(poll); 524 if (info) 525 add_debug_info_for_branch(info); // This isn't just debug info: 526 // it's the oop map 527 else 528 __ code_section()->relocate(pc(), rtype); 529 __ ldrw(zr, Address(rscratch1, off)); 530 __ bind(nope); 531 } 532 533 void LIR_Assembler::return_op(LIR_Opr result) { 534 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,"); 535 // Pop the stack before the safepoint code 536 __ remove_frame(initial_frame_size_in_bytes()); 537 address polling_page(os::get_polling_page()); 538 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type); 539 __ ret(lr); 540 } 541 542 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 543 address polling_page(os::get_polling_page()); 544 guarantee(info != NULL, "Shouldn't be NULL"); 545 assert(os::is_poll_address(polling_page), "should be"); 546 unsigned long off; 547 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off); 548 assert(off == 0, "must be"); 549 add_debug_info_for_branch(info); // This isn't just debug info: 550 // it's the oop map 551 __ read_polling_page(rscratch1, relocInfo::poll_type); 552 return __ offset(); 553 } 554 555 556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 557 if (from_reg == r31_sp) 558 from_reg = sp; 559 if (to_reg == r31_sp) 560 to_reg = sp; 561 __ mov(to_reg, from_reg); 562 } 563 564 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); } 565 566 567 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 568 assert(src->is_constant(), "should not call otherwise"); 569 assert(dest->is_register(), "should not call otherwise"); 570 LIR_Const* c = src->as_constant_ptr(); 571 572 switch (c->type()) { 573 case T_INT: { 574 assert(patch_code == lir_patch_none, "no patching handled here"); 575 __ movw(dest->as_register(), c->as_jint()); 576 break; 577 } 578 579 case T_ADDRESS: { 580 assert(patch_code == lir_patch_none, "no patching handled here"); 581 __ mov(dest->as_register(), c->as_jint()); 582 break; 583 } 584 585 case T_LONG: { 586 assert(patch_code == lir_patch_none, "no patching handled here"); 587 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong()); 588 break; 589 } 590 591 case T_OBJECT: { 592 if (patch_code == lir_patch_none) { 593 jobject2reg(c->as_jobject(), dest->as_register()); 594 } else { 595 jobject2reg_with_patching(dest->as_register(), info); 596 } 597 break; 598 } 599 600 case T_METADATA: { 601 if (patch_code != lir_patch_none) { 602 klass2reg_with_patching(dest->as_register(), info); 603 } else { 604 __ mov_metadata(dest->as_register(), c->as_metadata()); 605 } 606 break; 607 } 608 609 case T_FLOAT: { 610 if (__ operand_valid_for_float_immediate(c->as_jfloat())) { 611 __ fmovs(dest->as_float_reg(), (c->as_jfloat())); 612 } else { 613 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat()))); 614 __ ldrs(dest->as_float_reg(), Address(rscratch1)); 615 } 616 break; 617 } 618 619 case T_DOUBLE: { 620 if (__ operand_valid_for_float_immediate(c->as_jdouble())) { 621 __ fmovd(dest->as_double_reg(), (c->as_jdouble())); 622 } else { 623 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble()))); 624 __ ldrd(dest->as_double_reg(), Address(rscratch1)); 625 } 626 break; 627 } 628 629 default: 630 ShouldNotReachHere(); 631 } 632 } 633 634 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 635 LIR_Const* c = src->as_constant_ptr(); 636 switch (c->type()) { 637 case T_OBJECT: 638 { 639 if (! c->as_jobject()) 640 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 641 else { 642 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL); 643 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 644 } 645 } 646 break; 647 case T_ADDRESS: 648 { 649 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL); 650 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 651 } 652 case T_INT: 653 case T_FLOAT: 654 { 655 Register reg = zr; 656 if (c->as_jint_bits() == 0) 657 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 658 else { 659 __ movw(rscratch1, c->as_jint_bits()); 660 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix())); 661 } 662 } 663 break; 664 case T_LONG: 665 case T_DOUBLE: 666 { 667 Register reg = zr; 668 if (c->as_jlong_bits() == 0) 669 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(), 670 lo_word_offset_in_bytes)); 671 else { 672 __ mov(rscratch1, (intptr_t)c->as_jlong_bits()); 673 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(), 674 lo_word_offset_in_bytes)); 675 } 676 } 677 break; 678 default: 679 ShouldNotReachHere(); 680 } 681 } 682 683 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 684 assert(src->is_constant(), "should not call otherwise"); 685 LIR_Const* c = src->as_constant_ptr(); 686 LIR_Address* to_addr = dest->as_address_ptr(); 687 688 void (Assembler::* insn)(Register Rt, const Address &adr); 689 690 switch (type) { 691 case T_ADDRESS: 692 assert(c->as_jint() == 0, "should be"); 693 insn = &Assembler::str; 694 break; 695 case T_LONG: 696 assert(c->as_jlong() == 0, "should be"); 697 insn = &Assembler::str; 698 break; 699 case T_INT: 700 assert(c->as_jint() == 0, "should be"); 701 insn = &Assembler::strw; 702 break; 703 case T_OBJECT: 704 case T_ARRAY: 705 assert(c->as_jobject() == 0, "should be"); 706 if (UseCompressedOops && !wide) { 707 insn = &Assembler::strw; 708 } else { 709 insn = &Assembler::str; 710 } 711 break; 712 case T_CHAR: 713 case T_SHORT: 714 assert(c->as_jint() == 0, "should be"); 715 insn = &Assembler::strh; 716 break; 717 case T_BOOLEAN: 718 case T_BYTE: 719 assert(c->as_jint() == 0, "should be"); 720 insn = &Assembler::strb; 721 break; 722 default: 723 ShouldNotReachHere(); 724 insn = &Assembler::str; // unreachable 725 } 726 727 if (info) add_debug_info_for_null_check_here(info); 728 (_masm->*insn)(zr, as_Address(to_addr, rscratch1)); 729 } 730 731 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 732 assert(src->is_register(), "should not call otherwise"); 733 assert(dest->is_register(), "should not call otherwise"); 734 735 // move between cpu-registers 736 if (dest->is_single_cpu()) { 737 if (src->type() == T_LONG) { 738 // Can do LONG -> OBJECT 739 move_regs(src->as_register_lo(), dest->as_register()); 740 return; 741 } 742 assert(src->is_single_cpu(), "must match"); 743 if (src->type() == T_OBJECT) { 744 __ verify_oop(src->as_register()); 745 } 746 move_regs(src->as_register(), dest->as_register()); 747 748 } else if (dest->is_double_cpu()) { 749 if (src->type() == T_OBJECT || src->type() == T_ARRAY) { 750 // Surprising to me but we can see move of a long to t_object 751 __ verify_oop(src->as_register()); 752 move_regs(src->as_register(), dest->as_register_lo()); 753 return; 754 } 755 assert(src->is_double_cpu(), "must match"); 756 Register f_lo = src->as_register_lo(); 757 Register f_hi = src->as_register_hi(); 758 Register t_lo = dest->as_register_lo(); 759 Register t_hi = dest->as_register_hi(); 760 assert(f_hi == f_lo, "must be same"); 761 assert(t_hi == t_lo, "must be same"); 762 move_regs(f_lo, t_lo); 763 764 } else if (dest->is_single_fpu()) { 765 __ fmovs(dest->as_float_reg(), src->as_float_reg()); 766 767 } else if (dest->is_double_fpu()) { 768 __ fmovd(dest->as_double_reg(), src->as_double_reg()); 769 770 } else { 771 ShouldNotReachHere(); 772 } 773 } 774 775 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 776 if (src->is_single_cpu()) { 777 if (type == T_ARRAY || type == T_OBJECT) { 778 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 779 __ verify_oop(src->as_register()); 780 } else if (type == T_METADATA || type == T_DOUBLE) { 781 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 782 } else { 783 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 784 } 785 786 } else if (src->is_double_cpu()) { 787 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes); 788 __ str(src->as_register_lo(), dest_addr_LO); 789 790 } else if (src->is_single_fpu()) { 791 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix()); 792 __ strs(src->as_float_reg(), dest_addr); 793 794 } else if (src->is_double_fpu()) { 795 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix()); 796 __ strd(src->as_double_reg(), dest_addr); 797 798 } else { 799 ShouldNotReachHere(); 800 } 801 802 } 803 804 805 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) { 806 LIR_Address* to_addr = dest->as_address_ptr(); 807 PatchingStub* patch = NULL; 808 Register compressed_src = rscratch1; 809 810 if (patch_code != lir_patch_none) { 811 deoptimize_trap(info); 812 return; 813 } 814 815 if (type == T_ARRAY || type == T_OBJECT) { 816 __ verify_oop(src->as_register()); 817 818 if (UseCompressedOops && !wide) { 819 __ encode_heap_oop(compressed_src, src->as_register()); 820 } else { 821 compressed_src = src->as_register(); 822 } 823 } 824 825 int null_check_here = code_offset(); 826 switch (type) { 827 case T_FLOAT: { 828 __ strs(src->as_float_reg(), as_Address(to_addr)); 829 break; 830 } 831 832 case T_DOUBLE: { 833 __ strd(src->as_double_reg(), as_Address(to_addr)); 834 break; 835 } 836 837 case T_ARRAY: // fall through 838 case T_OBJECT: // fall through 839 if (UseCompressedOops && !wide) { 840 __ strw(compressed_src, as_Address(to_addr, rscratch2)); 841 } else { 842 __ str(compressed_src, as_Address(to_addr)); 843 } 844 break; 845 case T_METADATA: 846 // We get here to store a method pointer to the stack to pass to 847 // a dtrace runtime call. This can't work on 64 bit with 848 // compressed klass ptrs: T_METADATA can be a compressed klass 849 // ptr or a 64 bit method pointer. 850 ShouldNotReachHere(); 851 __ str(src->as_register(), as_Address(to_addr)); 852 break; 853 case T_ADDRESS: 854 __ str(src->as_register(), as_Address(to_addr)); 855 break; 856 case T_INT: 857 __ strw(src->as_register(), as_Address(to_addr)); 858 break; 859 860 case T_LONG: { 861 __ str(src->as_register_lo(), as_Address_lo(to_addr)); 862 break; 863 } 864 865 case T_BYTE: // fall through 866 case T_BOOLEAN: { 867 __ strb(src->as_register(), as_Address(to_addr)); 868 break; 869 } 870 871 case T_CHAR: // fall through 872 case T_SHORT: 873 __ strh(src->as_register(), as_Address(to_addr)); 874 break; 875 876 default: 877 ShouldNotReachHere(); 878 } 879 if (info != NULL) { 880 add_debug_info_for_null_check(null_check_here, info); 881 } 882 } 883 884 885 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 886 assert(src->is_stack(), "should not call otherwise"); 887 assert(dest->is_register(), "should not call otherwise"); 888 889 if (dest->is_single_cpu()) { 890 if (type == T_ARRAY || type == T_OBJECT) { 891 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 892 __ verify_oop(dest->as_register()); 893 } else if (type == T_METADATA) { 894 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 895 } else { 896 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 897 } 898 899 } else if (dest->is_double_cpu()) { 900 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes); 901 __ ldr(dest->as_register_lo(), src_addr_LO); 902 903 } else if (dest->is_single_fpu()) { 904 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix()); 905 __ ldrs(dest->as_float_reg(), src_addr); 906 907 } else if (dest->is_double_fpu()) { 908 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix()); 909 __ ldrd(dest->as_double_reg(), src_addr); 910 911 } else { 912 ShouldNotReachHere(); 913 } 914 } 915 916 917 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 918 address target = NULL; 919 relocInfo::relocType reloc_type = relocInfo::none; 920 921 switch (patching_id(info)) { 922 case PatchingStub::access_field_id: 923 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 924 reloc_type = relocInfo::section_word_type; 925 break; 926 case PatchingStub::load_klass_id: 927 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 928 reloc_type = relocInfo::metadata_type; 929 break; 930 case PatchingStub::load_mirror_id: 931 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 932 reloc_type = relocInfo::oop_type; 933 break; 934 case PatchingStub::load_appendix_id: 935 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 936 reloc_type = relocInfo::oop_type; 937 break; 938 default: ShouldNotReachHere(); 939 } 940 941 __ far_call(RuntimeAddress(target)); 942 add_call_info_here(info); 943 } 944 945 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 946 947 LIR_Opr temp; 948 if (type == T_LONG || type == T_DOUBLE) 949 temp = FrameMap::rscratch1_long_opr; 950 else 951 temp = FrameMap::rscratch1_opr; 952 953 stack2reg(src, temp, src->type()); 954 reg2stack(temp, dest, dest->type(), false); 955 } 956 957 958 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) { 959 LIR_Address* addr = src->as_address_ptr(); 960 LIR_Address* from_addr = src->as_address_ptr(); 961 962 if (addr->base()->type() == T_OBJECT) { 963 __ verify_oop(addr->base()->as_pointer_register()); 964 } 965 966 if (patch_code != lir_patch_none) { 967 deoptimize_trap(info); 968 return; 969 } 970 971 if (info != NULL) { 972 add_debug_info_for_null_check_here(info); 973 } 974 int null_check_here = code_offset(); 975 switch (type) { 976 case T_FLOAT: { 977 __ ldrs(dest->as_float_reg(), as_Address(from_addr)); 978 break; 979 } 980 981 case T_DOUBLE: { 982 __ ldrd(dest->as_double_reg(), as_Address(from_addr)); 983 break; 984 } 985 986 case T_ARRAY: // fall through 987 case T_OBJECT: // fall through 988 if (UseCompressedOops && !wide) { 989 __ ldrw(dest->as_register(), as_Address(from_addr)); 990 } else { 991 __ ldr(dest->as_register(), as_Address(from_addr)); 992 } 993 break; 994 case T_METADATA: 995 // We get here to store a method pointer to the stack to pass to 996 // a dtrace runtime call. This can't work on 64 bit with 997 // compressed klass ptrs: T_METADATA can be a compressed klass 998 // ptr or a 64 bit method pointer. 999 ShouldNotReachHere(); 1000 __ ldr(dest->as_register(), as_Address(from_addr)); 1001 break; 1002 case T_ADDRESS: 1003 // FIXME: OMG this is a horrible kludge. Any offset from an 1004 // address that matches klass_offset_in_bytes() will be loaded 1005 // as a word, not a long. 1006 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1007 __ ldrw(dest->as_register(), as_Address(from_addr)); 1008 } else { 1009 __ ldr(dest->as_register(), as_Address(from_addr)); 1010 } 1011 break; 1012 case T_INT: 1013 __ ldrw(dest->as_register(), as_Address(from_addr)); 1014 break; 1015 1016 case T_LONG: { 1017 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr)); 1018 break; 1019 } 1020 1021 case T_BYTE: 1022 __ ldrsb(dest->as_register(), as_Address(from_addr)); 1023 break; 1024 case T_BOOLEAN: { 1025 __ ldrb(dest->as_register(), as_Address(from_addr)); 1026 break; 1027 } 1028 1029 case T_CHAR: 1030 __ ldrh(dest->as_register(), as_Address(from_addr)); 1031 break; 1032 case T_SHORT: 1033 __ ldrsh(dest->as_register(), as_Address(from_addr)); 1034 break; 1035 1036 default: 1037 ShouldNotReachHere(); 1038 } 1039 1040 if (type == T_ARRAY || type == T_OBJECT) { 1041 if (UseCompressedOops && !wide) { 1042 __ decode_heap_oop(dest->as_register()); 1043 } 1044 __ verify_oop(dest->as_register()); 1045 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1046 if (UseCompressedClassPointers) { 1047 __ decode_klass_not_null(dest->as_register()); 1048 } 1049 } 1050 } 1051 1052 1053 int LIR_Assembler::array_element_size(BasicType type) const { 1054 int elem_size = type2aelembytes(type); 1055 return exact_log2(elem_size); 1056 } 1057 1058 void LIR_Assembler::arithmetic_idiv(LIR_Op3* op, bool is_irem) { 1059 Register Rdividend = op->in_opr1()->as_register(); 1060 Register Rdivisor = op->in_opr2()->as_register(); 1061 Register Rscratch = op->in_opr3()->as_register(); 1062 Register Rresult = op->result_opr()->as_register(); 1063 int divisor = -1; 1064 1065 /* 1066 TODO: For some reason, using the Rscratch that gets passed in is 1067 not possible because the register allocator does not see the tmp reg 1068 as used, and assignes it the same register as Rdividend. We use rscratch1 1069 instead. 1070 1071 assert(Rdividend != Rscratch, ""); 1072 assert(Rdivisor != Rscratch, ""); 1073 */ 1074 1075 if (Rdivisor == noreg && is_power_of_2(divisor)) { 1076 // convert division by a power of two into some shifts and logical operations 1077 } 1078 1079 __ corrected_idivl(Rresult, Rdividend, Rdivisor, is_irem, rscratch1); 1080 } 1081 1082 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1083 switch (op->code()) { 1084 case lir_idiv: 1085 arithmetic_idiv(op, false); 1086 break; 1087 case lir_irem: 1088 arithmetic_idiv(op, true); 1089 break; 1090 case lir_fmad: 1091 __ fmaddd(op->result_opr()->as_double_reg(), 1092 op->in_opr1()->as_double_reg(), 1093 op->in_opr2()->as_double_reg(), 1094 op->in_opr3()->as_double_reg()); 1095 break; 1096 case lir_fmaf: 1097 __ fmadds(op->result_opr()->as_float_reg(), 1098 op->in_opr1()->as_float_reg(), 1099 op->in_opr2()->as_float_reg(), 1100 op->in_opr3()->as_float_reg()); 1101 break; 1102 default: ShouldNotReachHere(); break; 1103 } 1104 } 1105 1106 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1107 #ifdef ASSERT 1108 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label"); 1109 if (op->block() != NULL) _branch_target_blocks.append(op->block()); 1110 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock()); 1111 #endif 1112 1113 if (op->cond() == lir_cond_always) { 1114 if (op->info() != NULL) add_debug_info_for_branch(op->info()); 1115 __ b(*(op->label())); 1116 } else { 1117 Assembler::Condition acond; 1118 if (op->code() == lir_cond_float_branch) { 1119 bool is_unordered = (op->ublock() == op->block()); 1120 // Assembler::EQ does not permit unordered branches, so we add 1121 // another branch here. Likewise, Assembler::NE does not permit 1122 // ordered branches. 1123 if (is_unordered && op->cond() == lir_cond_equal 1124 || !is_unordered && op->cond() == lir_cond_notEqual) 1125 __ br(Assembler::VS, *(op->ublock()->label())); 1126 switch(op->cond()) { 1127 case lir_cond_equal: acond = Assembler::EQ; break; 1128 case lir_cond_notEqual: acond = Assembler::NE; break; 1129 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break; 1130 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break; 1131 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break; 1132 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break; 1133 default: ShouldNotReachHere(); 1134 acond = Assembler::EQ; // unreachable 1135 } 1136 } else { 1137 switch (op->cond()) { 1138 case lir_cond_equal: acond = Assembler::EQ; break; 1139 case lir_cond_notEqual: acond = Assembler::NE; break; 1140 case lir_cond_less: acond = Assembler::LT; break; 1141 case lir_cond_lessEqual: acond = Assembler::LE; break; 1142 case lir_cond_greaterEqual: acond = Assembler::GE; break; 1143 case lir_cond_greater: acond = Assembler::GT; break; 1144 case lir_cond_belowEqual: acond = Assembler::LS; break; 1145 case lir_cond_aboveEqual: acond = Assembler::HS; break; 1146 default: ShouldNotReachHere(); 1147 acond = Assembler::EQ; // unreachable 1148 } 1149 } 1150 __ br(acond,*(op->label())); 1151 } 1152 } 1153 1154 1155 1156 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1157 LIR_Opr src = op->in_opr(); 1158 LIR_Opr dest = op->result_opr(); 1159 1160 switch (op->bytecode()) { 1161 case Bytecodes::_i2f: 1162 { 1163 __ scvtfws(dest->as_float_reg(), src->as_register()); 1164 break; 1165 } 1166 case Bytecodes::_i2d: 1167 { 1168 __ scvtfwd(dest->as_double_reg(), src->as_register()); 1169 break; 1170 } 1171 case Bytecodes::_l2d: 1172 { 1173 __ scvtfd(dest->as_double_reg(), src->as_register_lo()); 1174 break; 1175 } 1176 case Bytecodes::_l2f: 1177 { 1178 __ scvtfs(dest->as_float_reg(), src->as_register_lo()); 1179 break; 1180 } 1181 case Bytecodes::_f2d: 1182 { 1183 __ fcvts(dest->as_double_reg(), src->as_float_reg()); 1184 break; 1185 } 1186 case Bytecodes::_d2f: 1187 { 1188 __ fcvtd(dest->as_float_reg(), src->as_double_reg()); 1189 break; 1190 } 1191 case Bytecodes::_i2c: 1192 { 1193 __ ubfx(dest->as_register(), src->as_register(), 0, 16); 1194 break; 1195 } 1196 case Bytecodes::_i2l: 1197 { 1198 __ sxtw(dest->as_register_lo(), src->as_register()); 1199 break; 1200 } 1201 case Bytecodes::_i2s: 1202 { 1203 __ sxth(dest->as_register(), src->as_register()); 1204 break; 1205 } 1206 case Bytecodes::_i2b: 1207 { 1208 __ sxtb(dest->as_register(), src->as_register()); 1209 break; 1210 } 1211 case Bytecodes::_l2i: 1212 { 1213 _masm->block_comment("FIXME: This could be a no-op"); 1214 __ uxtw(dest->as_register(), src->as_register_lo()); 1215 break; 1216 } 1217 case Bytecodes::_d2l: 1218 { 1219 __ fcvtzd(dest->as_register_lo(), src->as_double_reg()); 1220 break; 1221 } 1222 case Bytecodes::_f2i: 1223 { 1224 __ fcvtzsw(dest->as_register(), src->as_float_reg()); 1225 break; 1226 } 1227 case Bytecodes::_f2l: 1228 { 1229 __ fcvtzs(dest->as_register_lo(), src->as_float_reg()); 1230 break; 1231 } 1232 case Bytecodes::_d2i: 1233 { 1234 __ fcvtzdw(dest->as_register(), src->as_double_reg()); 1235 break; 1236 } 1237 default: ShouldNotReachHere(); 1238 } 1239 } 1240 1241 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1242 if (op->init_check()) { 1243 __ ldrb(rscratch1, Address(op->klass()->as_register(), 1244 InstanceKlass::init_state_offset())); 1245 __ cmpw(rscratch1, InstanceKlass::fully_initialized); 1246 add_debug_info_for_null_check_here(op->stub()->info()); 1247 __ br(Assembler::NE, *op->stub()->entry()); 1248 } 1249 __ allocate_object(op->obj()->as_register(), 1250 op->tmp1()->as_register(), 1251 op->tmp2()->as_register(), 1252 op->header_size(), 1253 op->object_size(), 1254 op->klass()->as_register(), 1255 *op->stub()->entry()); 1256 __ bind(*op->stub()->continuation()); 1257 } 1258 1259 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1260 Register len = op->len()->as_register(); 1261 __ uxtw(len, len); 1262 1263 if (UseSlowPath || 1264 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || 1265 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { 1266 __ b(*op->stub()->entry()); 1267 } else { 1268 Register tmp1 = op->tmp1()->as_register(); 1269 Register tmp2 = op->tmp2()->as_register(); 1270 Register tmp3 = op->tmp3()->as_register(); 1271 if (len == tmp1) { 1272 tmp1 = tmp3; 1273 } else if (len == tmp2) { 1274 tmp2 = tmp3; 1275 } else if (len == tmp3) { 1276 // everything is ok 1277 } else { 1278 __ mov(tmp3, len); 1279 } 1280 __ allocate_array(op->obj()->as_register(), 1281 len, 1282 tmp1, 1283 tmp2, 1284 arrayOopDesc::header_size(op->type()), 1285 array_element_size(op->type()), 1286 op->klass()->as_register(), 1287 *op->stub()->entry()); 1288 } 1289 __ bind(*op->stub()->continuation()); 1290 } 1291 1292 void LIR_Assembler::type_profile_helper(Register mdo, 1293 ciMethodData *md, ciProfileData *data, 1294 Register recv, Label* update_done) { 1295 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1296 Label next_test; 1297 // See if the receiver is receiver[n]. 1298 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1299 __ ldr(rscratch1, Address(rscratch2)); 1300 __ cmp(recv, rscratch1); 1301 __ br(Assembler::NE, next_test); 1302 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); 1303 __ addptr(data_addr, DataLayout::counter_increment); 1304 __ b(*update_done); 1305 __ bind(next_test); 1306 } 1307 1308 // Didn't find receiver; find next empty slot and fill it in 1309 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1310 Label next_test; 1311 __ lea(rscratch2, 1312 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1313 Address recv_addr(rscratch2); 1314 __ ldr(rscratch1, recv_addr); 1315 __ cbnz(rscratch1, next_test); 1316 __ str(recv, recv_addr); 1317 __ mov(rscratch1, DataLayout::counter_increment); 1318 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)))); 1319 __ str(rscratch1, Address(rscratch2)); 1320 __ b(*update_done); 1321 __ bind(next_test); 1322 } 1323 } 1324 1325 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1326 // we always need a stub for the failure case. 1327 CodeStub* stub = op->stub(); 1328 Register obj = op->object()->as_register(); 1329 Register k_RInfo = op->tmp1()->as_register(); 1330 Register klass_RInfo = op->tmp2()->as_register(); 1331 Register dst = op->result_opr()->as_register(); 1332 ciKlass* k = op->klass(); 1333 Register Rtmp1 = noreg; 1334 1335 // check if it needs to be profiled 1336 ciMethodData* md; 1337 ciProfileData* data; 1338 1339 const bool should_profile = op->should_profile(); 1340 1341 if (should_profile) { 1342 ciMethod* method = op->profiled_method(); 1343 assert(method != NULL, "Should have method"); 1344 int bci = op->profiled_bci(); 1345 md = method->method_data_or_null(); 1346 assert(md != NULL, "Sanity"); 1347 data = md->bci_to_data(bci); 1348 assert(data != NULL, "need data for type check"); 1349 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1350 } 1351 Label profile_cast_success, profile_cast_failure; 1352 Label *success_target = should_profile ? &profile_cast_success : success; 1353 Label *failure_target = should_profile ? &profile_cast_failure : failure; 1354 1355 if (obj == k_RInfo) { 1356 k_RInfo = dst; 1357 } else if (obj == klass_RInfo) { 1358 klass_RInfo = dst; 1359 } 1360 if (k->is_loaded() && !UseCompressedClassPointers) { 1361 select_different_registers(obj, dst, k_RInfo, klass_RInfo); 1362 } else { 1363 Rtmp1 = op->tmp3()->as_register(); 1364 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1365 } 1366 1367 assert_different_registers(obj, k_RInfo, klass_RInfo); 1368 1369 if (should_profile) { 1370 Label not_null; 1371 __ cbnz(obj, not_null); 1372 // Object is null; update MDO and exit 1373 Register mdo = klass_RInfo; 1374 __ mov_metadata(mdo, md->constant_encoding()); 1375 Address data_addr 1376 = __ form_address(rscratch2, mdo, 1377 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()), 1378 LogBytesPerWord); 1379 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant()); 1380 __ ldr(rscratch1, data_addr); 1381 __ orr(rscratch1, rscratch1, header_bits); 1382 __ str(rscratch1, data_addr); 1383 __ b(*obj_is_null); 1384 __ bind(not_null); 1385 } else { 1386 __ cbz(obj, *obj_is_null); 1387 } 1388 1389 if (!k->is_loaded()) { 1390 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1391 } else { 1392 __ mov_metadata(k_RInfo, k->constant_encoding()); 1393 } 1394 __ verify_oop(obj); 1395 1396 if (op->fast_check()) { 1397 // get object class 1398 // not a safepoint as obj null check happens earlier 1399 __ load_klass(rscratch1, obj); 1400 __ cmp( rscratch1, k_RInfo); 1401 1402 __ br(Assembler::NE, *failure_target); 1403 // successful cast, fall through to profile or jump 1404 } else { 1405 // get object class 1406 // not a safepoint as obj null check happens earlier 1407 __ load_klass(klass_RInfo, obj); 1408 if (k->is_loaded()) { 1409 // See if we get an immediate positive hit 1410 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset()))); 1411 __ cmp(k_RInfo, rscratch1); 1412 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1413 __ br(Assembler::NE, *failure_target); 1414 // successful cast, fall through to profile or jump 1415 } else { 1416 // See if we get an immediate positive hit 1417 __ br(Assembler::EQ, *success_target); 1418 // check for self 1419 __ cmp(klass_RInfo, k_RInfo); 1420 __ br(Assembler::EQ, *success_target); 1421 1422 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1423 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1424 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1425 // result is a boolean 1426 __ cbzw(klass_RInfo, *failure_target); 1427 // successful cast, fall through to profile or jump 1428 } 1429 } else { 1430 // perform the fast part of the checking logic 1431 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1432 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1433 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1434 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1435 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1436 // result is a boolean 1437 __ cbz(k_RInfo, *failure_target); 1438 // successful cast, fall through to profile or jump 1439 } 1440 } 1441 if (should_profile) { 1442 Register mdo = klass_RInfo, recv = k_RInfo; 1443 __ bind(profile_cast_success); 1444 __ mov_metadata(mdo, md->constant_encoding()); 1445 __ load_klass(recv, obj); 1446 Label update_done; 1447 type_profile_helper(mdo, md, data, recv, success); 1448 __ b(*success); 1449 1450 __ bind(profile_cast_failure); 1451 __ mov_metadata(mdo, md->constant_encoding()); 1452 Address counter_addr 1453 = __ form_address(rscratch2, mdo, 1454 md->byte_offset_of_slot(data, CounterData::count_offset()), 1455 LogBytesPerWord); 1456 __ ldr(rscratch1, counter_addr); 1457 __ sub(rscratch1, rscratch1, DataLayout::counter_increment); 1458 __ str(rscratch1, counter_addr); 1459 __ b(*failure); 1460 } 1461 __ b(*success); 1462 } 1463 1464 1465 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1466 const bool should_profile = op->should_profile(); 1467 1468 LIR_Code code = op->code(); 1469 if (code == lir_store_check) { 1470 Register value = op->object()->as_register(); 1471 Register array = op->array()->as_register(); 1472 Register k_RInfo = op->tmp1()->as_register(); 1473 Register klass_RInfo = op->tmp2()->as_register(); 1474 Register Rtmp1 = op->tmp3()->as_register(); 1475 1476 CodeStub* stub = op->stub(); 1477 1478 // check if it needs to be profiled 1479 ciMethodData* md; 1480 ciProfileData* data; 1481 1482 if (should_profile) { 1483 ciMethod* method = op->profiled_method(); 1484 assert(method != NULL, "Should have method"); 1485 int bci = op->profiled_bci(); 1486 md = method->method_data_or_null(); 1487 assert(md != NULL, "Sanity"); 1488 data = md->bci_to_data(bci); 1489 assert(data != NULL, "need data for type check"); 1490 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1491 } 1492 Label profile_cast_success, profile_cast_failure, done; 1493 Label *success_target = should_profile ? &profile_cast_success : &done; 1494 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry(); 1495 1496 if (should_profile) { 1497 Label not_null; 1498 __ cbnz(value, not_null); 1499 // Object is null; update MDO and exit 1500 Register mdo = klass_RInfo; 1501 __ mov_metadata(mdo, md->constant_encoding()); 1502 Address data_addr 1503 = __ form_address(rscratch2, mdo, 1504 md->byte_offset_of_slot(data, DataLayout::header_offset()), 1505 LogBytesPerInt); 1506 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant()); 1507 __ ldrw(rscratch1, data_addr); 1508 __ orrw(rscratch1, rscratch1, header_bits); 1509 __ strw(rscratch1, data_addr); 1510 __ b(done); 1511 __ bind(not_null); 1512 } else { 1513 __ cbz(value, done); 1514 } 1515 1516 add_debug_info_for_null_check_here(op->info_for_exception()); 1517 __ load_klass(k_RInfo, array); 1518 __ load_klass(klass_RInfo, value); 1519 1520 // get instance klass (it's already uncompressed) 1521 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1522 // perform the fast part of the checking logic 1523 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1524 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1525 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1526 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1527 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1528 // result is a boolean 1529 __ cbzw(k_RInfo, *failure_target); 1530 // fall through to the success case 1531 1532 if (should_profile) { 1533 Register mdo = klass_RInfo, recv = k_RInfo; 1534 __ bind(profile_cast_success); 1535 __ mov_metadata(mdo, md->constant_encoding()); 1536 __ load_klass(recv, value); 1537 Label update_done; 1538 type_profile_helper(mdo, md, data, recv, &done); 1539 __ b(done); 1540 1541 __ bind(profile_cast_failure); 1542 __ mov_metadata(mdo, md->constant_encoding()); 1543 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1544 __ lea(rscratch2, counter_addr); 1545 __ ldr(rscratch1, Address(rscratch2)); 1546 __ sub(rscratch1, rscratch1, DataLayout::counter_increment); 1547 __ str(rscratch1, Address(rscratch2)); 1548 __ b(*stub->entry()); 1549 } 1550 1551 __ bind(done); 1552 } else if (code == lir_checkcast) { 1553 Register obj = op->object()->as_register(); 1554 Register dst = op->result_opr()->as_register(); 1555 Label success; 1556 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1557 __ bind(success); 1558 if (dst != obj) { 1559 __ mov(dst, obj); 1560 } 1561 } else if (code == lir_instanceof) { 1562 Register obj = op->object()->as_register(); 1563 Register dst = op->result_opr()->as_register(); 1564 Label success, failure, done; 1565 emit_typecheck_helper(op, &success, &failure, &failure); 1566 __ bind(failure); 1567 __ mov(dst, zr); 1568 __ b(done); 1569 __ bind(success); 1570 __ mov(dst, 1); 1571 __ bind(done); 1572 } else { 1573 ShouldNotReachHere(); 1574 } 1575 } 1576 1577 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { 1578 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); 1579 __ cset(rscratch1, Assembler::NE); 1580 __ membar(__ AnyAny); 1581 } 1582 1583 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { 1584 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); 1585 __ cset(rscratch1, Assembler::NE); 1586 __ membar(__ AnyAny); 1587 } 1588 1589 1590 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1591 assert(VM_Version::supports_cx8(), "wrong machine"); 1592 Register addr = as_reg(op->addr()); 1593 Register newval = as_reg(op->new_value()); 1594 Register cmpval = as_reg(op->cmp_value()); 1595 Label succeed, fail, around; 1596 1597 if (op->code() == lir_cas_obj) { 1598 if (UseCompressedOops) { 1599 Register t1 = op->tmp1()->as_register(); 1600 assert(op->tmp1()->is_valid(), "must be"); 1601 __ encode_heap_oop(t1, cmpval); 1602 cmpval = t1; 1603 __ encode_heap_oop(rscratch2, newval); 1604 newval = rscratch2; 1605 casw(addr, newval, cmpval); 1606 } else { 1607 casl(addr, newval, cmpval); 1608 } 1609 } else if (op->code() == lir_cas_int) { 1610 casw(addr, newval, cmpval); 1611 } else { 1612 casl(addr, newval, cmpval); 1613 } 1614 } 1615 1616 1617 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) { 1618 1619 Assembler::Condition acond, ncond; 1620 switch (condition) { 1621 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break; 1622 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break; 1623 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break; 1624 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break; 1625 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break; 1626 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break; 1627 case lir_cond_belowEqual: 1628 case lir_cond_aboveEqual: 1629 default: ShouldNotReachHere(); 1630 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable 1631 } 1632 1633 assert(result->is_single_cpu() || result->is_double_cpu(), 1634 "expect single register for result"); 1635 if (opr1->is_constant() && opr2->is_constant() 1636 && opr1->type() == T_INT && opr2->type() == T_INT) { 1637 jint val1 = opr1->as_jint(); 1638 jint val2 = opr2->as_jint(); 1639 if (val1 == 0 && val2 == 1) { 1640 __ cset(result->as_register(), ncond); 1641 return; 1642 } else if (val1 == 1 && val2 == 0) { 1643 __ cset(result->as_register(), acond); 1644 return; 1645 } 1646 } 1647 1648 if (opr1->is_constant() && opr2->is_constant() 1649 && opr1->type() == T_LONG && opr2->type() == T_LONG) { 1650 jlong val1 = opr1->as_jlong(); 1651 jlong val2 = opr2->as_jlong(); 1652 if (val1 == 0 && val2 == 1) { 1653 __ cset(result->as_register_lo(), ncond); 1654 return; 1655 } else if (val1 == 1 && val2 == 0) { 1656 __ cset(result->as_register_lo(), acond); 1657 return; 1658 } 1659 } 1660 1661 if (opr1->is_stack()) { 1662 stack2reg(opr1, FrameMap::rscratch1_opr, result->type()); 1663 opr1 = FrameMap::rscratch1_opr; 1664 } else if (opr1->is_constant()) { 1665 LIR_Opr tmp 1666 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr; 1667 const2reg(opr1, tmp, lir_patch_none, NULL); 1668 opr1 = tmp; 1669 } 1670 1671 if (opr2->is_stack()) { 1672 stack2reg(opr2, FrameMap::rscratch2_opr, result->type()); 1673 opr2 = FrameMap::rscratch2_opr; 1674 } else if (opr2->is_constant()) { 1675 LIR_Opr tmp 1676 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr; 1677 const2reg(opr2, tmp, lir_patch_none, NULL); 1678 opr2 = tmp; 1679 } 1680 1681 if (result->type() == T_LONG) 1682 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond); 1683 else 1684 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond); 1685 } 1686 1687 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 1688 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method"); 1689 1690 if (left->is_single_cpu()) { 1691 Register lreg = left->as_register(); 1692 Register dreg = as_reg(dest); 1693 1694 if (right->is_single_cpu()) { 1695 // cpu register - cpu register 1696 1697 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT, 1698 "should be"); 1699 Register rreg = right->as_register(); 1700 switch (code) { 1701 case lir_add: __ addw (dest->as_register(), lreg, rreg); break; 1702 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break; 1703 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break; 1704 default: ShouldNotReachHere(); 1705 } 1706 1707 } else if (right->is_double_cpu()) { 1708 Register rreg = right->as_register_lo(); 1709 // single_cpu + double_cpu: can happen with obj+long 1710 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1711 switch (code) { 1712 case lir_add: __ add(dreg, lreg, rreg); break; 1713 case lir_sub: __ sub(dreg, lreg, rreg); break; 1714 default: ShouldNotReachHere(); 1715 } 1716 } else if (right->is_constant()) { 1717 // cpu register - constant 1718 jlong c; 1719 1720 // FIXME. This is fugly: we really need to factor all this logic. 1721 switch(right->type()) { 1722 case T_LONG: 1723 c = right->as_constant_ptr()->as_jlong(); 1724 break; 1725 case T_INT: 1726 case T_ADDRESS: 1727 c = right->as_constant_ptr()->as_jint(); 1728 break; 1729 default: 1730 ShouldNotReachHere(); 1731 c = 0; // unreachable 1732 break; 1733 } 1734 1735 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1736 if (c == 0 && dreg == lreg) { 1737 COMMENT("effective nop elided"); 1738 return; 1739 } 1740 switch(left->type()) { 1741 case T_INT: 1742 switch (code) { 1743 case lir_add: __ addw(dreg, lreg, c); break; 1744 case lir_sub: __ subw(dreg, lreg, c); break; 1745 default: ShouldNotReachHere(); 1746 } 1747 break; 1748 case T_OBJECT: 1749 case T_ADDRESS: 1750 switch (code) { 1751 case lir_add: __ add(dreg, lreg, c); break; 1752 case lir_sub: __ sub(dreg, lreg, c); break; 1753 default: ShouldNotReachHere(); 1754 } 1755 break; 1756 ShouldNotReachHere(); 1757 } 1758 } else { 1759 ShouldNotReachHere(); 1760 } 1761 1762 } else if (left->is_double_cpu()) { 1763 Register lreg_lo = left->as_register_lo(); 1764 1765 if (right->is_double_cpu()) { 1766 // cpu register - cpu register 1767 Register rreg_lo = right->as_register_lo(); 1768 switch (code) { 1769 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1770 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1771 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1772 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break; 1773 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break; 1774 default: 1775 ShouldNotReachHere(); 1776 } 1777 1778 } else if (right->is_constant()) { 1779 jlong c = right->as_constant_ptr()->as_jlong_bits(); 1780 Register dreg = as_reg(dest); 1781 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1782 if (c == 0 && dreg == lreg_lo) { 1783 COMMENT("effective nop elided"); 1784 return; 1785 } 1786 switch (code) { 1787 case lir_add: __ add(dreg, lreg_lo, c); break; 1788 case lir_sub: __ sub(dreg, lreg_lo, c); break; 1789 default: 1790 ShouldNotReachHere(); 1791 } 1792 } else { 1793 ShouldNotReachHere(); 1794 } 1795 } else if (left->is_single_fpu()) { 1796 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register"); 1797 switch (code) { 1798 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1799 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1800 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1801 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1802 default: 1803 ShouldNotReachHere(); 1804 } 1805 } else if (left->is_double_fpu()) { 1806 if (right->is_double_fpu()) { 1807 // cpu register - cpu register 1808 switch (code) { 1809 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1810 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1811 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1812 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1813 default: 1814 ShouldNotReachHere(); 1815 } 1816 } else { 1817 if (right->is_constant()) { 1818 ShouldNotReachHere(); 1819 } 1820 ShouldNotReachHere(); 1821 } 1822 } else if (left->is_single_stack() || left->is_address()) { 1823 assert(left == dest, "left and dest must be equal"); 1824 ShouldNotReachHere(); 1825 } else { 1826 ShouldNotReachHere(); 1827 } 1828 } 1829 1830 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); } 1831 1832 1833 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { 1834 switch(code) { 1835 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break; 1836 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break; 1837 default : ShouldNotReachHere(); 1838 } 1839 } 1840 1841 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 1842 1843 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register"); 1844 Register Rleft = left->is_single_cpu() ? left->as_register() : 1845 left->as_register_lo(); 1846 if (dst->is_single_cpu()) { 1847 Register Rdst = dst->as_register(); 1848 if (right->is_constant()) { 1849 switch (code) { 1850 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break; 1851 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break; 1852 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break; 1853 default: ShouldNotReachHere(); break; 1854 } 1855 } else { 1856 Register Rright = right->is_single_cpu() ? right->as_register() : 1857 right->as_register_lo(); 1858 switch (code) { 1859 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break; 1860 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break; 1861 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break; 1862 default: ShouldNotReachHere(); break; 1863 } 1864 } 1865 } else { 1866 Register Rdst = dst->as_register_lo(); 1867 if (right->is_constant()) { 1868 switch (code) { 1869 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break; 1870 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break; 1871 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break; 1872 default: ShouldNotReachHere(); break; 1873 } 1874 } else { 1875 Register Rright = right->is_single_cpu() ? right->as_register() : 1876 right->as_register_lo(); 1877 switch (code) { 1878 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break; 1879 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break; 1880 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break; 1881 default: ShouldNotReachHere(); break; 1882 } 1883 } 1884 } 1885 } 1886 1887 1888 1889 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); } 1890 1891 1892 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 1893 if (opr1->is_constant() && opr2->is_single_cpu()) { 1894 // tableswitch 1895 Register reg = as_reg(opr2); 1896 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()]; 1897 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after); 1898 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) { 1899 Register reg1 = as_reg(opr1); 1900 if (opr2->is_single_cpu()) { 1901 // cpu register - cpu register 1902 Register reg2 = opr2->as_register(); 1903 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) { 1904 __ cmp(reg1, reg2); 1905 } else { 1906 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?"); 1907 __ cmpw(reg1, reg2); 1908 } 1909 return; 1910 } 1911 if (opr2->is_double_cpu()) { 1912 // cpu register - cpu register 1913 Register reg2 = opr2->as_register_lo(); 1914 __ cmp(reg1, reg2); 1915 return; 1916 } 1917 1918 if (opr2->is_constant()) { 1919 jlong imm; 1920 switch(opr2->type()) { 1921 case T_LONG: 1922 imm = opr2->as_constant_ptr()->as_jlong(); 1923 break; 1924 case T_INT: 1925 case T_ADDRESS: 1926 imm = opr2->as_constant_ptr()->as_jint(); 1927 break; 1928 case T_OBJECT: 1929 case T_ARRAY: 1930 imm = jlong(opr2->as_constant_ptr()->as_jobject()); 1931 break; 1932 default: 1933 ShouldNotReachHere(); 1934 imm = 0; // unreachable 1935 break; 1936 } 1937 1938 if (Assembler::operand_valid_for_add_sub_immediate(imm)) { 1939 if (type2aelembytes(opr1->type()) <= 4) 1940 __ cmpw(reg1, imm); 1941 else 1942 __ cmp(reg1, imm); 1943 return; 1944 } else { 1945 __ mov(rscratch1, imm); 1946 if (type2aelembytes(opr1->type()) <= 4) 1947 __ cmpw(reg1, rscratch1); 1948 else 1949 __ cmp(reg1, rscratch1); 1950 return; 1951 } 1952 } else 1953 ShouldNotReachHere(); 1954 } else if (opr1->is_single_fpu()) { 1955 FloatRegister reg1 = opr1->as_float_reg(); 1956 assert(opr2->is_single_fpu(), "expect single float register"); 1957 FloatRegister reg2 = opr2->as_float_reg(); 1958 __ fcmps(reg1, reg2); 1959 } else if (opr1->is_double_fpu()) { 1960 FloatRegister reg1 = opr1->as_double_reg(); 1961 assert(opr2->is_double_fpu(), "expect double float register"); 1962 FloatRegister reg2 = opr2->as_double_reg(); 1963 __ fcmpd(reg1, reg2); 1964 } else { 1965 ShouldNotReachHere(); 1966 } 1967 } 1968 1969 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){ 1970 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 1971 bool is_unordered_less = (code == lir_ucmp_fd2i); 1972 if (left->is_single_fpu()) { 1973 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register()); 1974 } else if (left->is_double_fpu()) { 1975 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register()); 1976 } else { 1977 ShouldNotReachHere(); 1978 } 1979 } else if (code == lir_cmp_l2i) { 1980 Label done; 1981 __ cmp(left->as_register_lo(), right->as_register_lo()); 1982 __ mov(dst->as_register(), (u_int64_t)-1L); 1983 __ br(Assembler::LT, done); 1984 __ csinc(dst->as_register(), zr, zr, Assembler::EQ); 1985 __ bind(done); 1986 } else { 1987 ShouldNotReachHere(); 1988 } 1989 } 1990 1991 1992 void LIR_Assembler::align_call(LIR_Code code) { } 1993 1994 1995 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 1996 address call = __ trampoline_call(Address(op->addr(), rtype)); 1997 if (call == NULL) { 1998 bailout("trampoline stub overflow"); 1999 return; 2000 } 2001 add_call_info(code_offset(), op->info()); 2002 } 2003 2004 2005 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 2006 address call = __ ic_call(op->addr()); 2007 if (call == NULL) { 2008 bailout("trampoline stub overflow"); 2009 return; 2010 } 2011 add_call_info(code_offset(), op->info()); 2012 } 2013 2014 2015 /* Currently, vtable-dispatch is only enabled for sparc platforms */ 2016 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) { 2017 ShouldNotReachHere(); 2018 } 2019 2020 2021 void LIR_Assembler::emit_static_call_stub() { 2022 address call_pc = __ pc(); 2023 address stub = __ start_a_stub(call_stub_size); 2024 if (stub == NULL) { 2025 bailout("static call stub overflow"); 2026 return; 2027 } 2028 2029 int start = __ offset(); 2030 2031 __ relocate(static_stub_Relocation::spec(call_pc)); 2032 __ mov_metadata(rmethod, (Metadata*)NULL); 2033 __ movptr(rscratch1, 0); 2034 __ br(rscratch1); 2035 2036 assert(__ offset() - start <= call_stub_size, "stub too big"); 2037 __ end_a_stub(); 2038 } 2039 2040 2041 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 2042 assert(exceptionOop->as_register() == r0, "must match"); 2043 assert(exceptionPC->as_register() == r3, "must match"); 2044 2045 // exception object is not added to oop map by LinearScan 2046 // (LinearScan assumes that no oops are in fixed registers) 2047 info->add_register_oop(exceptionOop); 2048 Runtime1::StubID unwind_id; 2049 2050 // get current pc information 2051 // pc is only needed if the method has an exception handler, the unwind code does not need it. 2052 int pc_for_athrow_offset = __ offset(); 2053 InternalAddress pc_for_athrow(__ pc()); 2054 __ adr(exceptionPC->as_register(), pc_for_athrow); 2055 add_call_info(pc_for_athrow_offset, info); // for exception handler 2056 2057 __ verify_not_null_oop(r0); 2058 // search an exception handler (r0: exception oop, r3: throwing pc) 2059 if (compilation()->has_fpu_code()) { 2060 unwind_id = Runtime1::handle_exception_id; 2061 } else { 2062 unwind_id = Runtime1::handle_exception_nofpu_id; 2063 } 2064 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 2065 2066 // FIXME: enough room for two byte trap ???? 2067 __ nop(); 2068 } 2069 2070 2071 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 2072 assert(exceptionOop->as_register() == r0, "must match"); 2073 2074 __ b(_unwind_handler_entry); 2075 } 2076 2077 2078 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 2079 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2080 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2081 2082 switch (left->type()) { 2083 case T_INT: { 2084 switch (code) { 2085 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break; 2086 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break; 2087 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break; 2088 default: 2089 ShouldNotReachHere(); 2090 break; 2091 } 2092 break; 2093 case T_LONG: 2094 case T_ADDRESS: 2095 case T_OBJECT: 2096 switch (code) { 2097 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break; 2098 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break; 2099 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break; 2100 default: 2101 ShouldNotReachHere(); 2102 break; 2103 } 2104 break; 2105 default: 2106 ShouldNotReachHere(); 2107 break; 2108 } 2109 } 2110 } 2111 2112 2113 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 2114 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2115 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2116 2117 switch (left->type()) { 2118 case T_INT: { 2119 switch (code) { 2120 case lir_shl: __ lslw (dreg, lreg, count); break; 2121 case lir_shr: __ asrw (dreg, lreg, count); break; 2122 case lir_ushr: __ lsrw (dreg, lreg, count); break; 2123 default: 2124 ShouldNotReachHere(); 2125 break; 2126 } 2127 break; 2128 case T_LONG: 2129 case T_ADDRESS: 2130 case T_OBJECT: 2131 switch (code) { 2132 case lir_shl: __ lsl (dreg, lreg, count); break; 2133 case lir_shr: __ asr (dreg, lreg, count); break; 2134 case lir_ushr: __ lsr (dreg, lreg, count); break; 2135 default: 2136 ShouldNotReachHere(); 2137 break; 2138 } 2139 break; 2140 default: 2141 ShouldNotReachHere(); 2142 break; 2143 } 2144 } 2145 } 2146 2147 2148 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 2149 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2150 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2151 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2152 __ str (r, Address(sp, offset_from_rsp_in_bytes)); 2153 } 2154 2155 2156 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 2157 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2158 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2159 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2160 __ mov (rscratch1, c); 2161 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2162 } 2163 2164 2165 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) { 2166 ShouldNotReachHere(); 2167 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2168 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2169 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2170 __ lea(rscratch1, __ constant_oop_address(o)); 2171 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2172 } 2173 2174 2175 // This code replaces a call to arraycopy; no exception may 2176 // be thrown in this code, they must be thrown in the System.arraycopy 2177 // activation frame; we could save some checks if this would not be the case 2178 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 2179 ciArrayKlass* default_type = op->expected_type(); 2180 Register src = op->src()->as_register(); 2181 Register dst = op->dst()->as_register(); 2182 Register src_pos = op->src_pos()->as_register(); 2183 Register dst_pos = op->dst_pos()->as_register(); 2184 Register length = op->length()->as_register(); 2185 Register tmp = op->tmp()->as_register(); 2186 2187 CodeStub* stub = op->stub(); 2188 int flags = op->flags(); 2189 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL; 2190 if (basic_type == T_ARRAY) basic_type = T_OBJECT; 2191 2192 // if we don't know anything, just go through the generic arraycopy 2193 if (default_type == NULL // || basic_type == T_OBJECT 2194 ) { 2195 Label done; 2196 assert(src == r1 && src_pos == r2, "mismatch in calling convention"); 2197 2198 // Save the arguments in case the generic arraycopy fails and we 2199 // have to fall back to the JNI stub 2200 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2201 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2202 __ str(src, Address(sp, 4*BytesPerWord)); 2203 2204 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy); 2205 address copyfunc_addr = StubRoutines::generic_arraycopy(); 2206 2207 // The arguments are in java calling convention so we shift them 2208 // to C convention 2209 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4); 2210 __ mov(c_rarg0, j_rarg0); 2211 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4); 2212 __ mov(c_rarg1, j_rarg1); 2213 assert_different_registers(c_rarg2, j_rarg3, j_rarg4); 2214 __ mov(c_rarg2, j_rarg2); 2215 assert_different_registers(c_rarg3, j_rarg4); 2216 __ mov(c_rarg3, j_rarg3); 2217 __ mov(c_rarg4, j_rarg4); 2218 if (copyfunc_addr == NULL) { // Use C version if stub was not generated 2219 __ mov(rscratch1, RuntimeAddress(C_entry)); 2220 __ blrt(rscratch1, 5, 0, 1); 2221 } else { 2222 #ifndef PRODUCT 2223 if (PrintC1Statistics) { 2224 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); 2225 } 2226 #endif 2227 __ far_call(RuntimeAddress(copyfunc_addr)); 2228 } 2229 2230 __ cbz(r0, *stub->continuation()); 2231 2232 // Reload values from the stack so they are where the stub 2233 // expects them. 2234 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2235 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2236 __ ldr(src, Address(sp, 4*BytesPerWord)); 2237 2238 if (copyfunc_addr != NULL) { 2239 // r0 is -1^K where K == partial copied count 2240 __ eonw(rscratch1, r0, 0); 2241 // adjust length down and src/end pos up by partial copied count 2242 __ subw(length, length, rscratch1); 2243 __ addw(src_pos, src_pos, rscratch1); 2244 __ addw(dst_pos, dst_pos, rscratch1); 2245 } 2246 __ b(*stub->entry()); 2247 2248 __ bind(*stub->continuation()); 2249 return; 2250 } 2251 2252 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point"); 2253 2254 int elem_size = type2aelembytes(basic_type); 2255 int shift_amount; 2256 int scale = exact_log2(elem_size); 2257 2258 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); 2259 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); 2260 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); 2261 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); 2262 2263 // test for NULL 2264 if (flags & LIR_OpArrayCopy::src_null_check) { 2265 __ cbz(src, *stub->entry()); 2266 } 2267 if (flags & LIR_OpArrayCopy::dst_null_check) { 2268 __ cbz(dst, *stub->entry()); 2269 } 2270 2271 // If the compiler was not able to prove that exact type of the source or the destination 2272 // of the arraycopy is an array type, check at runtime if the source or the destination is 2273 // an instance type. 2274 if (flags & LIR_OpArrayCopy::type_check) { 2275 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) { 2276 __ load_klass(tmp, dst); 2277 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2278 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2279 __ br(Assembler::GE, *stub->entry()); 2280 } 2281 2282 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) { 2283 __ load_klass(tmp, src); 2284 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2285 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2286 __ br(Assembler::GE, *stub->entry()); 2287 } 2288 } 2289 2290 // check if negative 2291 if (flags & LIR_OpArrayCopy::src_pos_positive_check) { 2292 __ cmpw(src_pos, 0); 2293 __ br(Assembler::LT, *stub->entry()); 2294 } 2295 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) { 2296 __ cmpw(dst_pos, 0); 2297 __ br(Assembler::LT, *stub->entry()); 2298 } 2299 2300 if (flags & LIR_OpArrayCopy::length_positive_check) { 2301 __ cmpw(length, 0); 2302 __ br(Assembler::LT, *stub->entry()); 2303 } 2304 2305 if (flags & LIR_OpArrayCopy::src_range_check) { 2306 __ addw(tmp, src_pos, length); 2307 __ ldrw(rscratch1, src_length_addr); 2308 __ cmpw(tmp, rscratch1); 2309 __ br(Assembler::HI, *stub->entry()); 2310 } 2311 if (flags & LIR_OpArrayCopy::dst_range_check) { 2312 __ addw(tmp, dst_pos, length); 2313 __ ldrw(rscratch1, dst_length_addr); 2314 __ cmpw(tmp, rscratch1); 2315 __ br(Assembler::HI, *stub->entry()); 2316 } 2317 2318 if (flags & LIR_OpArrayCopy::type_check) { 2319 // We don't know the array types are compatible 2320 if (basic_type != T_OBJECT) { 2321 // Simple test for basic type arrays 2322 if (UseCompressedClassPointers) { 2323 __ ldrw(tmp, src_klass_addr); 2324 __ ldrw(rscratch1, dst_klass_addr); 2325 __ cmpw(tmp, rscratch1); 2326 } else { 2327 __ ldr(tmp, src_klass_addr); 2328 __ ldr(rscratch1, dst_klass_addr); 2329 __ cmp(tmp, rscratch1); 2330 } 2331 __ br(Assembler::NE, *stub->entry()); 2332 } else { 2333 // For object arrays, if src is a sub class of dst then we can 2334 // safely do the copy. 2335 Label cont, slow; 2336 2337 #define PUSH(r1, r2) \ 2338 stp(r1, r2, __ pre(sp, -2 * wordSize)); 2339 2340 #define POP(r1, r2) \ 2341 ldp(r1, r2, __ post(sp, 2 * wordSize)); 2342 2343 __ PUSH(src, dst); 2344 2345 __ load_klass(src, src); 2346 __ load_klass(dst, dst); 2347 2348 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL); 2349 2350 __ PUSH(src, dst); 2351 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 2352 __ POP(src, dst); 2353 2354 __ cbnz(src, cont); 2355 2356 __ bind(slow); 2357 __ POP(src, dst); 2358 2359 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 2360 if (copyfunc_addr != NULL) { // use stub if available 2361 // src is not a sub class of dst so we have to do a 2362 // per-element check. 2363 2364 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 2365 if ((flags & mask) != mask) { 2366 // Check that at least both of them object arrays. 2367 assert(flags & mask, "one of the two should be known to be an object array"); 2368 2369 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 2370 __ load_klass(tmp, src); 2371 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 2372 __ load_klass(tmp, dst); 2373 } 2374 int lh_offset = in_bytes(Klass::layout_helper_offset()); 2375 Address klass_lh_addr(tmp, lh_offset); 2376 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 2377 __ ldrw(rscratch1, klass_lh_addr); 2378 __ mov(rscratch2, objArray_lh); 2379 __ eorw(rscratch1, rscratch1, rscratch2); 2380 __ cbnzw(rscratch1, *stub->entry()); 2381 } 2382 2383 // Spill because stubs can use any register they like and it's 2384 // easier to restore just those that we care about. 2385 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2386 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2387 __ str(src, Address(sp, 4*BytesPerWord)); 2388 2389 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2390 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2391 assert_different_registers(c_rarg0, dst, dst_pos, length); 2392 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2393 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2394 assert_different_registers(c_rarg1, dst, length); 2395 __ uxtw(c_rarg2, length); 2396 assert_different_registers(c_rarg2, dst); 2397 2398 __ load_klass(c_rarg4, dst); 2399 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset())); 2400 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset())); 2401 __ far_call(RuntimeAddress(copyfunc_addr)); 2402 2403 #ifndef PRODUCT 2404 if (PrintC1Statistics) { 2405 Label failed; 2406 __ cbnz(r0, failed); 2407 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt)); 2408 __ bind(failed); 2409 } 2410 #endif 2411 2412 __ cbz(r0, *stub->continuation()); 2413 2414 #ifndef PRODUCT 2415 if (PrintC1Statistics) { 2416 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt)); 2417 } 2418 #endif 2419 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1); 2420 2421 // Restore previously spilled arguments 2422 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2423 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2424 __ ldr(src, Address(sp, 4*BytesPerWord)); 2425 2426 // return value is -1^K where K is partial copied count 2427 __ eonw(rscratch1, r0, zr); 2428 // adjust length down and src/end pos up by partial copied count 2429 __ subw(length, length, rscratch1); 2430 __ addw(src_pos, src_pos, rscratch1); 2431 __ addw(dst_pos, dst_pos, rscratch1); 2432 } 2433 2434 __ b(*stub->entry()); 2435 2436 __ bind(cont); 2437 __ POP(src, dst); 2438 } 2439 } 2440 2441 #ifdef ASSERT 2442 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { 2443 // Sanity check the known type with the incoming class. For the 2444 // primitive case the types must match exactly with src.klass and 2445 // dst.klass each exactly matching the default type. For the 2446 // object array case, if no type check is needed then either the 2447 // dst type is exactly the expected type and the src type is a 2448 // subtype which we can't check or src is the same array as dst 2449 // but not necessarily exactly of type default_type. 2450 Label known_ok, halt; 2451 __ mov_metadata(tmp, default_type->constant_encoding()); 2452 if (UseCompressedClassPointers) { 2453 __ encode_klass_not_null(tmp); 2454 } 2455 2456 if (basic_type != T_OBJECT) { 2457 2458 if (UseCompressedClassPointers) { 2459 __ ldrw(rscratch1, dst_klass_addr); 2460 __ cmpw(tmp, rscratch1); 2461 } else { 2462 __ ldr(rscratch1, dst_klass_addr); 2463 __ cmp(tmp, rscratch1); 2464 } 2465 __ br(Assembler::NE, halt); 2466 if (UseCompressedClassPointers) { 2467 __ ldrw(rscratch1, src_klass_addr); 2468 __ cmpw(tmp, rscratch1); 2469 } else { 2470 __ ldr(rscratch1, src_klass_addr); 2471 __ cmp(tmp, rscratch1); 2472 } 2473 __ br(Assembler::EQ, known_ok); 2474 } else { 2475 if (UseCompressedClassPointers) { 2476 __ ldrw(rscratch1, dst_klass_addr); 2477 __ cmpw(tmp, rscratch1); 2478 } else { 2479 __ ldr(rscratch1, dst_klass_addr); 2480 __ cmp(tmp, rscratch1); 2481 } 2482 __ br(Assembler::EQ, known_ok); 2483 __ cmp(src, dst); 2484 __ br(Assembler::EQ, known_ok); 2485 } 2486 __ bind(halt); 2487 __ stop("incorrect type information in arraycopy"); 2488 __ bind(known_ok); 2489 } 2490 #endif 2491 2492 #ifndef PRODUCT 2493 if (PrintC1Statistics) { 2494 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type))); 2495 } 2496 #endif 2497 2498 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2499 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2500 assert_different_registers(c_rarg0, dst, dst_pos, length); 2501 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2502 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2503 assert_different_registers(c_rarg1, dst, length); 2504 __ uxtw(c_rarg2, length); 2505 assert_different_registers(c_rarg2, dst); 2506 2507 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 2508 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 2509 const char *name; 2510 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 2511 2512 CodeBlob *cb = CodeCache::find_blob(entry); 2513 if (cb) { 2514 __ far_call(RuntimeAddress(entry)); 2515 } else { 2516 __ call_VM_leaf(entry, 3); 2517 } 2518 2519 __ bind(*stub->continuation()); 2520 } 2521 2522 2523 2524 2525 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 2526 Register obj = op->obj_opr()->as_register(); // may not be an oop 2527 Register hdr = op->hdr_opr()->as_register(); 2528 Register lock = op->lock_opr()->as_register(); 2529 if (!UseFastLocking) { 2530 __ b(*op->stub()->entry()); 2531 } else if (op->code() == lir_lock) { 2532 Register scratch = noreg; 2533 if (UseBiasedLocking) { 2534 scratch = op->scratch_opr()->as_register(); 2535 } 2536 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2537 // add debug info for NullPointerException only if one is possible 2538 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry()); 2539 if (op->info() != NULL) { 2540 add_debug_info_for_null_check(null_check_offset, op->info()); 2541 } 2542 // done 2543 } else if (op->code() == lir_unlock) { 2544 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2545 __ unlock_object(hdr, obj, lock, *op->stub()->entry()); 2546 } else { 2547 Unimplemented(); 2548 } 2549 __ bind(*op->stub()->continuation()); 2550 } 2551 2552 2553 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 2554 ciMethod* method = op->profiled_method(); 2555 int bci = op->profiled_bci(); 2556 ciMethod* callee = op->profiled_callee(); 2557 2558 // Update counter for all call types 2559 ciMethodData* md = method->method_data_or_null(); 2560 assert(md != NULL, "Sanity"); 2561 ciProfileData* data = md->bci_to_data(bci); 2562 assert(data->is_CounterData(), "need CounterData for calls"); 2563 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 2564 Register mdo = op->mdo()->as_register(); 2565 __ mov_metadata(mdo, md->constant_encoding()); 2566 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 2567 Bytecodes::Code bc = method->java_code_at_bci(bci); 2568 const bool callee_is_static = callee->is_loaded() && callee->is_static(); 2569 // Perform additional virtual call profiling for invokevirtual and 2570 // invokeinterface bytecodes 2571 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) && 2572 !callee_is_static && // required for optimized MH invokes 2573 C1ProfileVirtualCalls) { 2574 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 2575 Register recv = op->recv()->as_register(); 2576 assert_different_registers(mdo, recv); 2577 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 2578 ciKlass* known_klass = op->known_holder(); 2579 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) { 2580 // We know the type that will be seen at this call site; we can 2581 // statically update the MethodData* rather than needing to do 2582 // dynamic tests on the receiver type 2583 2584 // NOTE: we should probably put a lock around this search to 2585 // avoid collisions by concurrent compilations 2586 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 2587 uint i; 2588 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2589 ciKlass* receiver = vc_data->receiver(i); 2590 if (known_klass->equals(receiver)) { 2591 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2592 __ addptr(data_addr, DataLayout::counter_increment); 2593 return; 2594 } 2595 } 2596 2597 // Receiver type not found in profile data; select an empty slot 2598 2599 // Note that this is less efficient than it should be because it 2600 // always does a write to the receiver part of the 2601 // VirtualCallData rather than just the first time 2602 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2603 ciKlass* receiver = vc_data->receiver(i); 2604 if (receiver == NULL) { 2605 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); 2606 __ mov_metadata(rscratch1, known_klass->constant_encoding()); 2607 __ lea(rscratch2, recv_addr); 2608 __ str(rscratch1, Address(rscratch2)); 2609 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2610 __ addptr(data_addr, DataLayout::counter_increment); 2611 return; 2612 } 2613 } 2614 } else { 2615 __ load_klass(recv, recv); 2616 Label update_done; 2617 type_profile_helper(mdo, md, data, recv, &update_done); 2618 // Receiver did not match any saved receiver and there is no empty row for it. 2619 // Increment total counter to indicate polymorphic case. 2620 __ addptr(counter_addr, DataLayout::counter_increment); 2621 2622 __ bind(update_done); 2623 } 2624 } else { 2625 // Static call 2626 __ addptr(counter_addr, DataLayout::counter_increment); 2627 } 2628 } 2629 2630 2631 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 2632 Unimplemented(); 2633 } 2634 2635 2636 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 2637 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 2638 } 2639 2640 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 2641 assert(op->crc()->is_single_cpu(), "crc must be register"); 2642 assert(op->val()->is_single_cpu(), "byte value must be register"); 2643 assert(op->result_opr()->is_single_cpu(), "result must be register"); 2644 Register crc = op->crc()->as_register(); 2645 Register val = op->val()->as_register(); 2646 Register res = op->result_opr()->as_register(); 2647 2648 assert_different_registers(val, crc, res); 2649 unsigned long offset; 2650 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset); 2651 if (offset) __ add(res, res, offset); 2652 2653 __ ornw(crc, zr, crc); // ~crc 2654 __ update_byte_crc32(crc, val, res); 2655 __ ornw(res, zr, crc); // ~crc 2656 } 2657 2658 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 2659 COMMENT("emit_profile_type {"); 2660 Register obj = op->obj()->as_register(); 2661 Register tmp = op->tmp()->as_pointer_register(); 2662 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 2663 ciKlass* exact_klass = op->exact_klass(); 2664 intptr_t current_klass = op->current_klass(); 2665 bool not_null = op->not_null(); 2666 bool no_conflict = op->no_conflict(); 2667 2668 Label update, next, none; 2669 2670 bool do_null = !not_null; 2671 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 2672 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 2673 2674 assert(do_null || do_update, "why are we here?"); 2675 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 2676 assert(mdo_addr.base() != rscratch1, "wrong register"); 2677 2678 __ verify_oop(obj); 2679 2680 if (tmp != obj) { 2681 __ mov(tmp, obj); 2682 } 2683 if (do_null) { 2684 __ cbnz(tmp, update); 2685 if (!TypeEntries::was_null_seen(current_klass)) { 2686 __ ldr(rscratch2, mdo_addr); 2687 __ orr(rscratch2, rscratch2, TypeEntries::null_seen); 2688 __ str(rscratch2, mdo_addr); 2689 } 2690 if (do_update) { 2691 #ifndef ASSERT 2692 __ b(next); 2693 } 2694 #else 2695 __ b(next); 2696 } 2697 } else { 2698 __ cbnz(tmp, update); 2699 __ stop("unexpected null obj"); 2700 #endif 2701 } 2702 2703 __ bind(update); 2704 2705 if (do_update) { 2706 #ifdef ASSERT 2707 if (exact_klass != NULL) { 2708 Label ok; 2709 __ load_klass(tmp, tmp); 2710 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2711 __ eor(rscratch1, tmp, rscratch1); 2712 __ cbz(rscratch1, ok); 2713 __ stop("exact klass and actual klass differ"); 2714 __ bind(ok); 2715 } 2716 #endif 2717 if (!no_conflict) { 2718 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) { 2719 if (exact_klass != NULL) { 2720 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2721 } else { 2722 __ load_klass(tmp, tmp); 2723 } 2724 2725 __ ldr(rscratch2, mdo_addr); 2726 __ eor(tmp, tmp, rscratch2); 2727 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2728 // klass seen before, nothing to do. The unknown bit may have been 2729 // set already but no need to check. 2730 __ cbz(rscratch1, next); 2731 2732 __ andr(rscratch1, tmp, TypeEntries::type_unknown); 2733 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore. 2734 2735 if (TypeEntries::is_type_none(current_klass)) { 2736 __ cbz(rscratch2, none); 2737 __ cmp(rscratch2, TypeEntries::null_seen); 2738 __ br(Assembler::EQ, none); 2739 // There is a chance that the checks above (re-reading profiling 2740 // data from memory) fail if another thread has just set the 2741 // profiling to this obj's klass 2742 __ dmb(Assembler::ISHLD); 2743 __ ldr(rscratch2, mdo_addr); 2744 __ eor(tmp, tmp, rscratch2); 2745 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2746 __ cbz(rscratch1, next); 2747 } 2748 } else { 2749 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 2750 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 2751 2752 __ ldr(tmp, mdo_addr); 2753 __ andr(rscratch1, tmp, TypeEntries::type_unknown); 2754 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore. 2755 } 2756 2757 // different than before. Cannot keep accurate profile. 2758 __ ldr(rscratch2, mdo_addr); 2759 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown); 2760 __ str(rscratch2, mdo_addr); 2761 2762 if (TypeEntries::is_type_none(current_klass)) { 2763 __ b(next); 2764 2765 __ bind(none); 2766 // first time here. Set profile type. 2767 __ str(tmp, mdo_addr); 2768 } 2769 } else { 2770 // There's a single possible klass at this profile point 2771 assert(exact_klass != NULL, "should be"); 2772 if (TypeEntries::is_type_none(current_klass)) { 2773 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2774 __ ldr(rscratch2, mdo_addr); 2775 __ eor(tmp, tmp, rscratch2); 2776 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2777 __ cbz(rscratch1, next); 2778 #ifdef ASSERT 2779 { 2780 Label ok; 2781 __ ldr(rscratch1, mdo_addr); 2782 __ cbz(rscratch1, ok); 2783 __ cmp(rscratch1, TypeEntries::null_seen); 2784 __ br(Assembler::EQ, ok); 2785 // may have been set by another thread 2786 __ dmb(Assembler::ISHLD); 2787 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2788 __ ldr(rscratch2, mdo_addr); 2789 __ eor(rscratch2, rscratch1, rscratch2); 2790 __ andr(rscratch2, rscratch2, TypeEntries::type_mask); 2791 __ cbz(rscratch2, ok); 2792 2793 __ stop("unexpected profiling mismatch"); 2794 __ bind(ok); 2795 } 2796 #endif 2797 // first time here. Set profile type. 2798 __ ldr(tmp, mdo_addr); 2799 } else { 2800 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 2801 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 2802 2803 __ ldr(tmp, mdo_addr); 2804 __ andr(rscratch1, tmp, TypeEntries::type_unknown); 2805 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore. 2806 2807 __ orr(tmp, tmp, TypeEntries::type_unknown); 2808 __ str(tmp, mdo_addr); 2809 // FIXME: Write barrier needed here? 2810 } 2811 } 2812 2813 __ bind(next); 2814 } 2815 COMMENT("} emit_profile_type"); 2816 } 2817 2818 2819 void LIR_Assembler::align_backward_branch_target() { 2820 } 2821 2822 2823 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) { 2824 if (left->is_single_cpu()) { 2825 assert(dest->is_single_cpu(), "expect single result reg"); 2826 __ negw(dest->as_register(), left->as_register()); 2827 } else if (left->is_double_cpu()) { 2828 assert(dest->is_double_cpu(), "expect double result reg"); 2829 __ neg(dest->as_register_lo(), left->as_register_lo()); 2830 } else if (left->is_single_fpu()) { 2831 assert(dest->is_single_fpu(), "expect single float result reg"); 2832 __ fnegs(dest->as_float_reg(), left->as_float_reg()); 2833 } else { 2834 assert(left->is_double_fpu(), "expect double float operand reg"); 2835 assert(dest->is_double_fpu(), "expect double float result reg"); 2836 __ fnegd(dest->as_double_reg(), left->as_double_reg()); 2837 } 2838 } 2839 2840 2841 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) { 2842 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr())); 2843 } 2844 2845 2846 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 2847 assert(!tmp->is_valid(), "don't need temporary"); 2848 2849 CodeBlob *cb = CodeCache::find_blob(dest); 2850 if (cb) { 2851 __ far_call(RuntimeAddress(dest)); 2852 } else { 2853 __ mov(rscratch1, RuntimeAddress(dest)); 2854 int len = args->length(); 2855 int type = 0; 2856 if (! result->is_illegal()) { 2857 switch (result->type()) { 2858 case T_VOID: 2859 type = 0; 2860 break; 2861 case T_INT: 2862 case T_LONG: 2863 case T_OBJECT: 2864 type = 1; 2865 break; 2866 case T_FLOAT: 2867 type = 2; 2868 break; 2869 case T_DOUBLE: 2870 type = 3; 2871 break; 2872 default: 2873 ShouldNotReachHere(); 2874 break; 2875 } 2876 } 2877 int num_gpargs = 0; 2878 int num_fpargs = 0; 2879 for (int i = 0; i < args->length(); i++) { 2880 LIR_Opr arg = args->at(i); 2881 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) { 2882 num_fpargs++; 2883 } else { 2884 num_gpargs++; 2885 } 2886 } 2887 __ blrt(rscratch1, num_gpargs, num_fpargs, type); 2888 } 2889 2890 if (info != NULL) { 2891 add_call_info_here(info); 2892 } 2893 __ maybe_isb(); 2894 } 2895 2896 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 2897 if (dest->is_address() || src->is_address()) { 2898 move_op(src, dest, type, lir_patch_none, info, 2899 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false); 2900 } else { 2901 ShouldNotReachHere(); 2902 } 2903 } 2904 2905 #ifdef ASSERT 2906 // emit run-time assertion 2907 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 2908 assert(op->code() == lir_assert, "must be"); 2909 2910 if (op->in_opr1()->is_valid()) { 2911 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 2912 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 2913 } else { 2914 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 2915 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 2916 } 2917 2918 Label ok; 2919 if (op->condition() != lir_cond_always) { 2920 Assembler::Condition acond = Assembler::AL; 2921 switch (op->condition()) { 2922 case lir_cond_equal: acond = Assembler::EQ; break; 2923 case lir_cond_notEqual: acond = Assembler::NE; break; 2924 case lir_cond_less: acond = Assembler::LT; break; 2925 case lir_cond_lessEqual: acond = Assembler::LE; break; 2926 case lir_cond_greaterEqual: acond = Assembler::GE; break; 2927 case lir_cond_greater: acond = Assembler::GT; break; 2928 case lir_cond_belowEqual: acond = Assembler::LS; break; 2929 case lir_cond_aboveEqual: acond = Assembler::HS; break; 2930 default: ShouldNotReachHere(); 2931 } 2932 __ br(acond, ok); 2933 } 2934 if (op->halt()) { 2935 const char* str = __ code_string(op->msg()); 2936 __ stop(str); 2937 } else { 2938 breakpoint(); 2939 } 2940 __ bind(ok); 2941 } 2942 #endif 2943 2944 #ifndef PRODUCT 2945 #define COMMENT(x) do { __ block_comment(x); } while (0) 2946 #else 2947 #define COMMENT(x) 2948 #endif 2949 2950 void LIR_Assembler::membar() { 2951 COMMENT("membar"); 2952 __ membar(MacroAssembler::AnyAny); 2953 } 2954 2955 void LIR_Assembler::membar_acquire() { 2956 __ membar(Assembler::LoadLoad|Assembler::LoadStore); 2957 } 2958 2959 void LIR_Assembler::membar_release() { 2960 __ membar(Assembler::LoadStore|Assembler::StoreStore); 2961 } 2962 2963 void LIR_Assembler::membar_loadload() { 2964 __ membar(Assembler::LoadLoad); 2965 } 2966 2967 void LIR_Assembler::membar_storestore() { 2968 __ membar(MacroAssembler::StoreStore); 2969 } 2970 2971 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); } 2972 2973 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); } 2974 2975 void LIR_Assembler::on_spin_wait() { 2976 Unimplemented(); 2977 } 2978 2979 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 2980 __ mov(result_reg->as_register(), rthread); 2981 } 2982 2983 2984 void LIR_Assembler::peephole(LIR_List *lir) { 2985 #if 0 2986 if (tableswitch_count >= max_tableswitches) 2987 return; 2988 2989 /* 2990 This finite-state automaton recognizes sequences of compare-and- 2991 branch instructions. We will turn them into a tableswitch. You 2992 could argue that C1 really shouldn't be doing this sort of 2993 optimization, but without it the code is really horrible. 2994 */ 2995 2996 enum { start_s, cmp1_s, beq_s, cmp_s } state; 2997 int first_key, last_key = -2147483648; 2998 int next_key = 0; 2999 int start_insn = -1; 3000 int last_insn = -1; 3001 Register reg = noreg; 3002 LIR_Opr reg_opr; 3003 state = start_s; 3004 3005 LIR_OpList* inst = lir->instructions_list(); 3006 for (int i = 0; i < inst->length(); i++) { 3007 LIR_Op* op = inst->at(i); 3008 switch (state) { 3009 case start_s: 3010 first_key = -1; 3011 start_insn = i; 3012 switch (op->code()) { 3013 case lir_cmp: 3014 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3015 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3016 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3017 && opr2->is_constant() 3018 && opr2->type() == T_INT) { 3019 reg_opr = opr1; 3020 reg = opr1->as_register(); 3021 first_key = opr2->as_constant_ptr()->as_jint(); 3022 next_key = first_key + 1; 3023 state = cmp_s; 3024 goto next_state; 3025 } 3026 break; 3027 } 3028 break; 3029 case cmp_s: 3030 switch (op->code()) { 3031 case lir_branch: 3032 if (op->as_OpBranch()->cond() == lir_cond_equal) { 3033 state = beq_s; 3034 last_insn = i; 3035 goto next_state; 3036 } 3037 } 3038 state = start_s; 3039 break; 3040 case beq_s: 3041 switch (op->code()) { 3042 case lir_cmp: { 3043 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3044 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3045 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3046 && opr1->as_register() == reg 3047 && opr2->is_constant() 3048 && opr2->type() == T_INT 3049 && opr2->as_constant_ptr()->as_jint() == next_key) { 3050 last_key = next_key; 3051 next_key++; 3052 state = cmp_s; 3053 goto next_state; 3054 } 3055 } 3056 } 3057 last_key = next_key; 3058 state = start_s; 3059 break; 3060 default: 3061 assert(false, "impossible state"); 3062 } 3063 if (state == start_s) { 3064 if (first_key < last_key - 5L && reg != noreg) { 3065 { 3066 // printf("found run register %d starting at insn %d low value %d high value %d\n", 3067 // reg->encoding(), 3068 // start_insn, first_key, last_key); 3069 // for (int i = 0; i < inst->length(); i++) { 3070 // inst->at(i)->print(); 3071 // tty->print("\n"); 3072 // } 3073 // tty->print("\n"); 3074 } 3075 3076 struct tableswitch *sw = &switches[tableswitch_count]; 3077 sw->_insn_index = start_insn, sw->_first_key = first_key, 3078 sw->_last_key = last_key, sw->_reg = reg; 3079 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after)); 3080 { 3081 // Insert the new table of branches 3082 int offset = last_insn; 3083 for (int n = first_key; n < last_key; n++) { 3084 inst->insert_before 3085 (last_insn + 1, 3086 new LIR_OpBranch(lir_cond_always, T_ILLEGAL, 3087 inst->at(offset)->as_OpBranch()->label())); 3088 offset -= 2, i++; 3089 } 3090 } 3091 // Delete all the old compare-and-branch instructions 3092 for (int n = first_key; n < last_key; n++) { 3093 inst->remove_at(start_insn); 3094 inst->remove_at(start_insn); 3095 } 3096 // Insert the tableswitch instruction 3097 inst->insert_before(start_insn, 3098 new LIR_Op2(lir_cmp, lir_cond_always, 3099 LIR_OprFact::intConst(tableswitch_count), 3100 reg_opr)); 3101 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches)); 3102 tableswitch_count++; 3103 } 3104 reg = noreg; 3105 last_key = -2147483648; 3106 } 3107 next_state: 3108 ; 3109 } 3110 #endif 3111 } 3112 3113 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { 3114 Address addr = as_Address(src->as_address_ptr(), noreg); 3115 BasicType type = src->type(); 3116 bool is_oop = type == T_OBJECT || type == T_ARRAY; 3117 3118 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr); 3119 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr); 3120 3121 switch(type) { 3122 case T_INT: 3123 xchg = &MacroAssembler::atomic_xchgalw; 3124 add = &MacroAssembler::atomic_addalw; 3125 break; 3126 case T_LONG: 3127 xchg = &MacroAssembler::atomic_xchgal; 3128 add = &MacroAssembler::atomic_addal; 3129 break; 3130 case T_OBJECT: 3131 case T_ARRAY: 3132 if (UseCompressedOops) { 3133 xchg = &MacroAssembler::atomic_xchgalw; 3134 add = &MacroAssembler::atomic_addalw; 3135 } else { 3136 xchg = &MacroAssembler::atomic_xchgal; 3137 add = &MacroAssembler::atomic_addal; 3138 } 3139 break; 3140 default: 3141 ShouldNotReachHere(); 3142 xchg = &MacroAssembler::atomic_xchgal; 3143 add = &MacroAssembler::atomic_addal; // unreachable 3144 } 3145 3146 switch (code) { 3147 case lir_xadd: 3148 { 3149 RegisterOrConstant inc; 3150 Register tmp = as_reg(tmp_op); 3151 Register dst = as_reg(dest); 3152 if (data->is_constant()) { 3153 inc = RegisterOrConstant(as_long(data)); 3154 assert_different_registers(dst, addr.base(), tmp, 3155 rscratch1, rscratch2); 3156 } else { 3157 inc = RegisterOrConstant(as_reg(data)); 3158 assert_different_registers(inc.as_register(), dst, addr.base(), tmp, 3159 rscratch1, rscratch2); 3160 } 3161 __ lea(tmp, addr); 3162 (_masm->*add)(dst, inc, tmp); 3163 break; 3164 } 3165 case lir_xchg: 3166 { 3167 Register tmp = tmp_op->as_register(); 3168 Register obj = as_reg(data); 3169 Register dst = as_reg(dest); 3170 if (is_oop && UseCompressedOops) { 3171 __ encode_heap_oop(rscratch2, obj); 3172 obj = rscratch2; 3173 } 3174 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst); 3175 __ lea(tmp, addr); 3176 (_masm->*xchg)(dst, obj, tmp); 3177 if (is_oop && UseCompressedOops) { 3178 __ decode_heap_oop(dst); 3179 } 3180 } 3181 break; 3182 default: 3183 ShouldNotReachHere(); 3184 } 3185 __ membar(__ AnyAny); 3186 } 3187 3188 #undef __