1 /*
   2  * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInstance.hpp"
  35 #include "gc/shared/barrierSet.hpp"
  36 #include "gc/shared/cardTableModRefBS.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "nativeInst_x86.hpp"
  39 #include "oops/objArrayKlass.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "vmreg_x86.inline.hpp"
  42 
  43 
  44 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  45 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  46 // fast versions of NegF/NegD and AbsF/AbsD.
  47 
  48 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  49 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  50   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  51   // of 128-bits operands for SSE instructions.
  52   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  53   // Store the value to a 128-bits operand.
  54   operand[0] = lo;
  55   operand[1] = hi;
  56   return operand;
  57 }
  58 
  59 // Buffer for 128-bits masks used by SSE instructions.
  60 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  61 
  62 // Static initialization during VM startup.
  63 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));
  64 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));
  65 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
  66 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
  67 
  68 
  69 
  70 NEEDS_CLEANUP // remove this definitions ?
  71 const Register IC_Klass    = rax;   // where the IC klass is cached
  72 const Register SYNC_header = rax;   // synchronization header
  73 const Register SHIFT_count = rcx;   // where count for shift operations must be
  74 
  75 #define __ _masm->
  76 
  77 
  78 static void select_different_registers(Register preserve,
  79                                        Register extra,
  80                                        Register &tmp1,
  81                                        Register &tmp2) {
  82   if (tmp1 == preserve) {
  83     assert_different_registers(tmp1, tmp2, extra);
  84     tmp1 = extra;
  85   } else if (tmp2 == preserve) {
  86     assert_different_registers(tmp1, tmp2, extra);
  87     tmp2 = extra;
  88   }
  89   assert_different_registers(preserve, tmp1, tmp2);
  90 }
  91 
  92 
  93 
  94 static void select_different_registers(Register preserve,
  95                                        Register extra,
  96                                        Register &tmp1,
  97                                        Register &tmp2,
  98                                        Register &tmp3) {
  99   if (tmp1 == preserve) {
 100     assert_different_registers(tmp1, tmp2, tmp3, extra);
 101     tmp1 = extra;
 102   } else if (tmp2 == preserve) {
 103     assert_different_registers(tmp1, tmp2, tmp3, extra);
 104     tmp2 = extra;
 105   } else if (tmp3 == preserve) {
 106     assert_different_registers(tmp1, tmp2, tmp3, extra);
 107     tmp3 = extra;
 108   }
 109   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 110 }
 111 
 112 
 113 
 114 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 115   if (opr->is_constant()) {
 116     LIR_Const* constant = opr->as_constant_ptr();
 117     switch (constant->type()) {
 118       case T_INT: {
 119         return true;
 120       }
 121 
 122       default:
 123         return false;
 124     }
 125   }
 126   return false;
 127 }
 128 
 129 
 130 LIR_Opr LIR_Assembler::receiverOpr() {
 131   return FrameMap::receiver_opr;
 132 }
 133 
 134 LIR_Opr LIR_Assembler::osrBufferPointer() {
 135   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 136 }
 137 
 138 //--------------fpu register translations-----------------------
 139 
 140 
 141 address LIR_Assembler::float_constant(float f) {
 142   address const_addr = __ float_constant(f);
 143   if (const_addr == NULL) {
 144     bailout("const section overflow");
 145     return __ code()->consts()->start();
 146   } else {
 147     return const_addr;
 148   }
 149 }
 150 
 151 
 152 address LIR_Assembler::double_constant(double d) {
 153   address const_addr = __ double_constant(d);
 154   if (const_addr == NULL) {
 155     bailout("const section overflow");
 156     return __ code()->consts()->start();
 157   } else {
 158     return const_addr;
 159   }
 160 }
 161 
 162 
 163 void LIR_Assembler::set_24bit_FPU() {
 164   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
 165 }
 166 
 167 void LIR_Assembler::reset_FPU() {
 168   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 169 }
 170 
 171 void LIR_Assembler::fpop() {
 172   __ fpop();
 173 }
 174 
 175 void LIR_Assembler::fxch(int i) {
 176   __ fxch(i);
 177 }
 178 
 179 void LIR_Assembler::fld(int i) {
 180   __ fld_s(i);
 181 }
 182 
 183 void LIR_Assembler::ffree(int i) {
 184   __ ffree(i);
 185 }
 186 
 187 void LIR_Assembler::breakpoint() {
 188   __ int3();
 189 }
 190 
 191 void LIR_Assembler::push(LIR_Opr opr) {
 192   if (opr->is_single_cpu()) {
 193     __ push_reg(opr->as_register());
 194   } else if (opr->is_double_cpu()) {
 195     NOT_LP64(__ push_reg(opr->as_register_hi()));
 196     __ push_reg(opr->as_register_lo());
 197   } else if (opr->is_stack()) {
 198     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 199   } else if (opr->is_constant()) {
 200     LIR_Const* const_opr = opr->as_constant_ptr();
 201     if (const_opr->type() == T_OBJECT) {
 202       __ push_oop(const_opr->as_jobject());
 203     } else if (const_opr->type() == T_INT) {
 204       __ push_jint(const_opr->as_jint());
 205     } else {
 206       ShouldNotReachHere();
 207     }
 208 
 209   } else {
 210     ShouldNotReachHere();
 211   }
 212 }
 213 
 214 void LIR_Assembler::pop(LIR_Opr opr) {
 215   if (opr->is_single_cpu()) {
 216     __ pop_reg(opr->as_register());
 217   } else {
 218     ShouldNotReachHere();
 219   }
 220 }
 221 
 222 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 223   return addr->base()->is_illegal() && addr->index()->is_illegal();
 224 }
 225 
 226 //-------------------------------------------
 227 
 228 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 229   return as_Address(addr, rscratch1);
 230 }
 231 
 232 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 233   if (addr->base()->is_illegal()) {
 234     assert(addr->index()->is_illegal(), "must be illegal too");
 235     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 236     if (! __ reachable(laddr)) {
 237       __ movptr(tmp, laddr.addr());
 238       Address res(tmp, 0);
 239       return res;
 240     } else {
 241       return __ as_Address(laddr);
 242     }
 243   }
 244 
 245   Register base = addr->base()->as_pointer_register();
 246 
 247   if (addr->index()->is_illegal()) {
 248     return Address( base, addr->disp());
 249   } else if (addr->index()->is_cpu_register()) {
 250     Register index = addr->index()->as_pointer_register();
 251     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 252   } else if (addr->index()->is_constant()) {
 253     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 254     assert(Assembler::is_simm32(addr_offset), "must be");
 255 
 256     return Address(base, addr_offset);
 257   } else {
 258     Unimplemented();
 259     return Address();
 260   }
 261 }
 262 
 263 
 264 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 265   Address base = as_Address(addr);
 266   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 267 }
 268 
 269 
 270 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 271   return as_Address(addr);
 272 }
 273 
 274 
 275 void LIR_Assembler::osr_entry() {
 276   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 277   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 278   ValueStack* entry_state = osr_entry->state();
 279   int number_of_locks = entry_state->locks_size();
 280 
 281   // we jump here if osr happens with the interpreter
 282   // state set up to continue at the beginning of the
 283   // loop that triggered osr - in particular, we have
 284   // the following registers setup:
 285   //
 286   // rcx: osr buffer
 287   //
 288 
 289   // build frame
 290   ciMethod* m = compilation()->method();
 291   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 292 
 293   // OSR buffer is
 294   //
 295   // locals[nlocals-1..0]
 296   // monitors[0..number_of_locks]
 297   //
 298   // locals is a direct copy of the interpreter frame so in the osr buffer
 299   // so first slot in the local array is the last local from the interpreter
 300   // and last slot is local[0] (receiver) from the interpreter
 301   //
 302   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 303   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 304   // in the interpreter frame (the method lock if a sync method)
 305 
 306   // Initialize monitors in the compiled activation.
 307   //   rcx: pointer to osr buffer
 308   //
 309   // All other registers are dead at this point and the locals will be
 310   // copied into place by code emitted in the IR.
 311 
 312   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 313   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 314     int monitor_offset = BytesPerWord * method()->max_locals() +
 315       (2 * BytesPerWord) * (number_of_locks - 1);
 316     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 317     // the OSR buffer using 2 word entries: first the lock and then
 318     // the oop.
 319     for (int i = 0; i < number_of_locks; i++) {
 320       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 321 #ifdef ASSERT
 322       // verify the interpreter's monitor has a non-null object
 323       {
 324         Label L;
 325         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD);
 326         __ jcc(Assembler::notZero, L);
 327         __ stop("locked object is NULL");
 328         __ bind(L);
 329       }
 330 #endif
 331       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 332       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 333       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 334       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 335     }
 336   }
 337 }
 338 
 339 
 340 // inline cache check; done before the frame is built.
 341 int LIR_Assembler::check_icache() {
 342   Register receiver = FrameMap::receiver_opr->as_register();
 343   Register ic_klass = IC_Klass;
 344   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
 345   const bool do_post_padding = VerifyOops || UseCompressedClassPointers;
 346   if (!do_post_padding) {
 347     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
 348     __ align(CodeEntryAlignment, __ offset() + ic_cmp_size);
 349   }
 350   int offset = __ offset();
 351   __ inline_cache_check(receiver, IC_Klass);
 352   assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
 353   if (do_post_padding) {
 354     // force alignment after the cache check.
 355     // It's been verified to be aligned if !VerifyOops
 356     __ align(CodeEntryAlignment);
 357   }
 358   return offset;
 359 }
 360 
 361 
 362 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 363   jobject o = NULL;
 364   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 365   __ movoop(reg, o);
 366   patching_epilog(patch, lir_patch_normal, reg, info);
 367 }
 368 
 369 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 370   Metadata* o = NULL;
 371   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 372   __ mov_metadata(reg, o);
 373   patching_epilog(patch, lir_patch_normal, reg, info);
 374 }
 375 
 376 // This specifies the rsp decrement needed to build the frame
 377 int LIR_Assembler::initial_frame_size_in_bytes() const {
 378   // if rounding, must let FrameMap know!
 379 
 380   // The frame_map records size in slots (32bit word)
 381 
 382   // subtract two words to account for return address and link
 383   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 384 }
 385 
 386 
 387 int LIR_Assembler::emit_exception_handler() {
 388   // if the last instruction is a call (typically to do a throw which
 389   // is coming at the end after block reordering) the return address
 390   // must still point into the code area in order to avoid assertion
 391   // failures when searching for the corresponding bci => add a nop
 392   // (was bug 5/14/1999 - gri)
 393   __ nop();
 394 
 395   // generate code for exception handler
 396   address handler_base = __ start_a_stub(exception_handler_size);
 397   if (handler_base == NULL) {
 398     // not enough space left for the handler
 399     bailout("exception handler overflow");
 400     return -1;
 401   }
 402 
 403   int offset = code_offset();
 404 
 405   // the exception oop and pc are in rax, and rdx
 406   // no other registers need to be preserved, so invalidate them
 407   __ invalidate_registers(false, true, true, false, true, true);
 408 
 409   // check that there is really an exception
 410   __ verify_not_null_oop(rax);
 411 
 412   // search an exception handler (rax: exception oop, rdx: throwing pc)
 413   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 414   __ should_not_reach_here();
 415   guarantee(code_offset() - offset <= exception_handler_size, "overflow");
 416   __ end_a_stub();
 417 
 418   return offset;
 419 }
 420 
 421 
 422 // Emit the code to remove the frame from the stack in the exception
 423 // unwind path.
 424 int LIR_Assembler::emit_unwind_handler() {
 425 #ifndef PRODUCT
 426   if (CommentedAssembly) {
 427     _masm->block_comment("Unwind handler");
 428   }
 429 #endif
 430 
 431   int offset = code_offset();
 432 
 433   // Fetch the exception from TLS and clear out exception related thread state
 434   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 435   NOT_LP64(__ get_thread(rsi));
 436   __ movptr(rax, Address(thread, JavaThread::exception_oop_offset()));
 437   __ movptr(Address(thread, JavaThread::exception_oop_offset()), (intptr_t)NULL_WORD);
 438   __ movptr(Address(thread, JavaThread::exception_pc_offset()), (intptr_t)NULL_WORD);
 439 
 440   __ bind(_unwind_handler_entry);
 441   __ verify_not_null_oop(rax);
 442   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 443     __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)
 444   }
 445 
 446   // Preform needed unlocking
 447   MonitorExitStub* stub = NULL;
 448   if (method()->is_synchronized()) {
 449     monitor_address(0, FrameMap::rax_opr);
 450     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 451     __ unlock_object(rdi, rsi, rax, *stub->entry());
 452     __ bind(*stub->continuation());
 453   }
 454 
 455   if (compilation()->env()->dtrace_method_probes()) {
 456 #ifdef _LP64
 457     __ mov(rdi, r15_thread);
 458     __ mov_metadata(rsi, method()->constant_encoding());
 459 #else
 460     __ get_thread(rax);
 461     __ movptr(Address(rsp, 0), rax);
 462     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 463 #endif
 464     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 465   }
 466 
 467   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 468     __ mov(rax, rbx);  // Restore the exception
 469   }
 470 
 471   // remove the activation and dispatch to the unwind handler
 472   __ remove_frame(initial_frame_size_in_bytes());
 473   __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 474 
 475   // Emit the slow path assembly
 476   if (stub != NULL) {
 477     stub->emit_code(this);
 478   }
 479 
 480   return offset;
 481 }
 482 
 483 
 484 int LIR_Assembler::emit_deopt_handler() {
 485   // if the last instruction is a call (typically to do a throw which
 486   // is coming at the end after block reordering) the return address
 487   // must still point into the code area in order to avoid assertion
 488   // failures when searching for the corresponding bci => add a nop
 489   // (was bug 5/14/1999 - gri)
 490   __ nop();
 491 
 492   // generate code for exception handler
 493   address handler_base = __ start_a_stub(deopt_handler_size);
 494   if (handler_base == NULL) {
 495     // not enough space left for the handler
 496     bailout("deopt handler overflow");
 497     return -1;
 498   }
 499 
 500   int offset = code_offset();
 501   InternalAddress here(__ pc());
 502 
 503   __ pushptr(here.addr());
 504   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 505   guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
 506   __ end_a_stub();
 507 
 508   return offset;
 509 }
 510 
 511 
 512 void LIR_Assembler::return_op(LIR_Opr result) {
 513   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 514   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 515     assert(result->fpu() == 0, "result must already be on TOS");
 516   }
 517 
 518   // Pop the stack before the safepoint code
 519   __ remove_frame(initial_frame_size_in_bytes());
 520 
 521   bool result_is_oop = result->is_valid() ? result->is_oop() : false;
 522 
 523   // Note: we do not need to round double result; float result has the right precision
 524   // the poll sets the condition code, but no data registers
 525   AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 526 
 527   if (Assembler::is_polling_page_far()) {
 528     __ lea(rscratch1, polling_page);
 529     __ relocate(relocInfo::poll_return_type);
 530     __ testl(rax, Address(rscratch1, 0));
 531   } else {
 532     __ testl(rax, polling_page);
 533   }
 534   __ ret(0);
 535 }
 536 
 537 
 538 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 539   AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_type);
 540   guarantee(info != NULL, "Shouldn't be NULL");
 541   int offset = __ offset();
 542   if (Assembler::is_polling_page_far()) {
 543     __ lea(rscratch1, polling_page);
 544     offset = __ offset();
 545     add_debug_info_for_branch(info);
 546     __ relocate(relocInfo::poll_type);
 547     __ testl(rax, Address(rscratch1, 0));
 548   } else {
 549     add_debug_info_for_branch(info);
 550     __ testl(rax, polling_page);
 551   }
 552   return offset;
 553 }
 554 
 555 
 556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 557   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 558 }
 559 
 560 void LIR_Assembler::swap_reg(Register a, Register b) {
 561   __ xchgptr(a, b);
 562 }
 563 
 564 
 565 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 566   assert(src->is_constant(), "should not call otherwise");
 567   assert(dest->is_register(), "should not call otherwise");
 568   LIR_Const* c = src->as_constant_ptr();
 569 
 570   switch (c->type()) {
 571     case T_INT: {
 572       assert(patch_code == lir_patch_none, "no patching handled here");
 573       __ movl(dest->as_register(), c->as_jint());
 574       break;
 575     }
 576 
 577     case T_ADDRESS: {
 578       assert(patch_code == lir_patch_none, "no patching handled here");
 579       __ movptr(dest->as_register(), c->as_jint());
 580       break;
 581     }
 582 
 583     case T_LONG: {
 584       assert(patch_code == lir_patch_none, "no patching handled here");
 585 #ifdef _LP64
 586       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 587 #else
 588       __ movptr(dest->as_register_lo(), c->as_jint_lo());
 589       __ movptr(dest->as_register_hi(), c->as_jint_hi());
 590 #endif // _LP64
 591       break;
 592     }
 593 
 594     case T_OBJECT: {
 595       if (patch_code != lir_patch_none) {
 596         jobject2reg_with_patching(dest->as_register(), info);
 597       } else {
 598         __ movoop(dest->as_register(), c->as_jobject());
 599       }
 600       break;
 601     }
 602 
 603     case T_METADATA: {
 604       if (patch_code != lir_patch_none) {
 605         klass2reg_with_patching(dest->as_register(), info);
 606       } else {
 607         __ mov_metadata(dest->as_register(), c->as_metadata());
 608       }
 609       break;
 610     }
 611 
 612     case T_FLOAT: {
 613       if (dest->is_single_xmm()) {
 614         if (c->is_zero_float()) {
 615           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 616         } else {
 617           __ movflt(dest->as_xmm_float_reg(),
 618                    InternalAddress(float_constant(c->as_jfloat())));
 619         }
 620       } else {
 621         assert(dest->is_single_fpu(), "must be");
 622         assert(dest->fpu_regnr() == 0, "dest must be TOS");
 623         if (c->is_zero_float()) {
 624           __ fldz();
 625         } else if (c->is_one_float()) {
 626           __ fld1();
 627         } else {
 628           __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
 629         }
 630       }
 631       break;
 632     }
 633 
 634     case T_DOUBLE: {
 635       if (dest->is_double_xmm()) {
 636         if (c->is_zero_double()) {
 637           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 638         } else {
 639           __ movdbl(dest->as_xmm_double_reg(),
 640                     InternalAddress(double_constant(c->as_jdouble())));
 641         }
 642       } else {
 643         assert(dest->is_double_fpu(), "must be");
 644         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
 645         if (c->is_zero_double()) {
 646           __ fldz();
 647         } else if (c->is_one_double()) {
 648           __ fld1();
 649         } else {
 650           __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
 651         }
 652       }
 653       break;
 654     }
 655 
 656     default:
 657       ShouldNotReachHere();
 658   }
 659 }
 660 
 661 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 662   assert(src->is_constant(), "should not call otherwise");
 663   assert(dest->is_stack(), "should not call otherwise");
 664   LIR_Const* c = src->as_constant_ptr();
 665 
 666   switch (c->type()) {
 667     case T_INT:  // fall through
 668     case T_FLOAT:
 669       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 670       break;
 671 
 672     case T_ADDRESS:
 673       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 674       break;
 675 
 676     case T_OBJECT:
 677       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject());
 678       break;
 679 
 680     case T_LONG:  // fall through
 681     case T_DOUBLE:
 682 #ifdef _LP64
 683       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 684                                             lo_word_offset_in_bytes), (intptr_t)c->as_jlong_bits());
 685 #else
 686       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 687                                               lo_word_offset_in_bytes), c->as_jint_lo_bits());
 688       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 689                                               hi_word_offset_in_bytes), c->as_jint_hi_bits());
 690 #endif // _LP64
 691       break;
 692 
 693     default:
 694       ShouldNotReachHere();
 695   }
 696 }
 697 
 698 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 699   assert(src->is_constant(), "should not call otherwise");
 700   assert(dest->is_address(), "should not call otherwise");
 701   LIR_Const* c = src->as_constant_ptr();
 702   LIR_Address* addr = dest->as_address_ptr();
 703 
 704   int null_check_here = code_offset();
 705   switch (type) {
 706     case T_INT:    // fall through
 707     case T_FLOAT:
 708       __ movl(as_Address(addr), c->as_jint_bits());
 709       break;
 710 
 711     case T_ADDRESS:
 712       __ movptr(as_Address(addr), c->as_jint_bits());
 713       break;
 714 
 715     case T_OBJECT:  // fall through
 716     case T_ARRAY:
 717       if (c->as_jobject() == NULL) {
 718         if (UseCompressedOops && !wide) {
 719           __ movl(as_Address(addr), (int32_t)NULL_WORD);
 720         } else {
 721 #ifdef _LP64
 722           __ xorptr(rscratch1, rscratch1);
 723           null_check_here = code_offset();
 724           __ movptr(as_Address(addr), rscratch1);
 725 #else
 726           __ movptr(as_Address(addr), NULL_WORD);
 727 #endif
 728         }
 729       } else {
 730         if (is_literal_address(addr)) {
 731           ShouldNotReachHere();
 732           __ movoop(as_Address(addr, noreg), c->as_jobject());
 733         } else {
 734 #ifdef _LP64
 735           __ movoop(rscratch1, c->as_jobject());
 736           if (UseCompressedOops && !wide) {
 737             __ encode_heap_oop(rscratch1);
 738             null_check_here = code_offset();
 739             __ movl(as_Address_lo(addr), rscratch1);
 740           } else {
 741             null_check_here = code_offset();
 742             __ movptr(as_Address_lo(addr), rscratch1);
 743           }
 744 #else
 745           __ movoop(as_Address(addr), c->as_jobject());
 746 #endif
 747         }
 748       }
 749       break;
 750 
 751     case T_LONG:    // fall through
 752     case T_DOUBLE:
 753 #ifdef _LP64
 754       if (is_literal_address(addr)) {
 755         ShouldNotReachHere();
 756         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 757       } else {
 758         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 759         null_check_here = code_offset();
 760         __ movptr(as_Address_lo(addr), r10);
 761       }
 762 #else
 763       // Always reachable in 32bit so this doesn't produce useless move literal
 764       __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
 765       __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
 766 #endif // _LP64
 767       break;
 768 
 769     case T_BOOLEAN: // fall through
 770     case T_BYTE:
 771       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 772       break;
 773 
 774     case T_CHAR:    // fall through
 775     case T_SHORT:
 776       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 777       break;
 778 
 779     default:
 780       ShouldNotReachHere();
 781   };
 782 
 783   if (info != NULL) {
 784     add_debug_info_for_null_check(null_check_here, info);
 785   }
 786 }
 787 
 788 
 789 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 790   assert(src->is_register(), "should not call otherwise");
 791   assert(dest->is_register(), "should not call otherwise");
 792 
 793   // move between cpu-registers
 794   if (dest->is_single_cpu()) {
 795 #ifdef _LP64
 796     if (src->type() == T_LONG) {
 797       // Can do LONG -> OBJECT
 798       move_regs(src->as_register_lo(), dest->as_register());
 799       return;
 800     }
 801 #endif
 802     assert(src->is_single_cpu(), "must match");
 803     if (src->type() == T_OBJECT) {
 804       __ verify_oop(src->as_register());
 805     }
 806     move_regs(src->as_register(), dest->as_register());
 807 
 808   } else if (dest->is_double_cpu()) {
 809 #ifdef _LP64
 810     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 811       // Surprising to me but we can see move of a long to t_object
 812       __ verify_oop(src->as_register());
 813       move_regs(src->as_register(), dest->as_register_lo());
 814       return;
 815     }
 816 #endif
 817     assert(src->is_double_cpu(), "must match");
 818     Register f_lo = src->as_register_lo();
 819     Register f_hi = src->as_register_hi();
 820     Register t_lo = dest->as_register_lo();
 821     Register t_hi = dest->as_register_hi();
 822 #ifdef _LP64
 823     assert(f_hi == f_lo, "must be same");
 824     assert(t_hi == t_lo, "must be same");
 825     move_regs(f_lo, t_lo);
 826 #else
 827     assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
 828 
 829 
 830     if (f_lo == t_hi && f_hi == t_lo) {
 831       swap_reg(f_lo, f_hi);
 832     } else if (f_hi == t_lo) {
 833       assert(f_lo != t_hi, "overwriting register");
 834       move_regs(f_hi, t_hi);
 835       move_regs(f_lo, t_lo);
 836     } else {
 837       assert(f_hi != t_lo, "overwriting register");
 838       move_regs(f_lo, t_lo);
 839       move_regs(f_hi, t_hi);
 840     }
 841 #endif // LP64
 842 
 843     // special moves from fpu-register to xmm-register
 844     // necessary for method results
 845   } else if (src->is_single_xmm() && !dest->is_single_xmm()) {
 846     __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
 847     __ fld_s(Address(rsp, 0));
 848   } else if (src->is_double_xmm() && !dest->is_double_xmm()) {
 849     __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
 850     __ fld_d(Address(rsp, 0));
 851   } else if (dest->is_single_xmm() && !src->is_single_xmm()) {
 852     __ fstp_s(Address(rsp, 0));
 853     __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
 854   } else if (dest->is_double_xmm() && !src->is_double_xmm()) {
 855     __ fstp_d(Address(rsp, 0));
 856     __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
 857 
 858     // move between xmm-registers
 859   } else if (dest->is_single_xmm()) {
 860     assert(src->is_single_xmm(), "must match");
 861     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 862   } else if (dest->is_double_xmm()) {
 863     assert(src->is_double_xmm(), "must match");
 864     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 865 
 866     // move between fpu-registers (no instruction necessary because of fpu-stack)
 867   } else if (dest->is_single_fpu() || dest->is_double_fpu()) {
 868     assert(src->is_single_fpu() || src->is_double_fpu(), "must match");
 869     assert(src->fpu() == dest->fpu(), "currently should be nothing to do");
 870   } else {
 871     ShouldNotReachHere();
 872   }
 873 }
 874 
 875 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 876   assert(src->is_register(), "should not call otherwise");
 877   assert(dest->is_stack(), "should not call otherwise");
 878 
 879   if (src->is_single_cpu()) {
 880     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 881     if (type == T_OBJECT || type == T_ARRAY) {
 882       __ verify_oop(src->as_register());
 883       __ movptr (dst, src->as_register());
 884     } else if (type == T_METADATA) {
 885       __ movptr (dst, src->as_register());
 886     } else {
 887       __ movl (dst, src->as_register());
 888     }
 889 
 890   } else if (src->is_double_cpu()) {
 891     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 892     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 893     __ movptr (dstLO, src->as_register_lo());
 894     NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
 895 
 896   } else if (src->is_single_xmm()) {
 897     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 898     __ movflt(dst_addr, src->as_xmm_float_reg());
 899 
 900   } else if (src->is_double_xmm()) {
 901     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 902     __ movdbl(dst_addr, src->as_xmm_double_reg());
 903 
 904   } else if (src->is_single_fpu()) {
 905     assert(src->fpu_regnr() == 0, "argument must be on TOS");
 906     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 907     if (pop_fpu_stack)     __ fstp_s (dst_addr);
 908     else                   __ fst_s  (dst_addr);
 909 
 910   } else if (src->is_double_fpu()) {
 911     assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
 912     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 913     if (pop_fpu_stack)     __ fstp_d (dst_addr);
 914     else                   __ fst_d  (dst_addr);
 915 
 916   } else {
 917     ShouldNotReachHere();
 918   }
 919 }
 920 
 921 
 922 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 923   LIR_Address* to_addr = dest->as_address_ptr();
 924   PatchingStub* patch = NULL;
 925   Register compressed_src = rscratch1;
 926 
 927   if (type == T_ARRAY || type == T_OBJECT) {
 928     __ verify_oop(src->as_register());
 929 #ifdef _LP64
 930     if (UseCompressedOops && !wide) {
 931       __ movptr(compressed_src, src->as_register());
 932       __ encode_heap_oop(compressed_src);
 933       if (patch_code != lir_patch_none) {
 934         info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
 935       }
 936     }
 937 #endif
 938   }
 939 
 940   if (patch_code != lir_patch_none) {
 941     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 942     Address toa = as_Address(to_addr);
 943     assert(toa.disp() != 0, "must have");
 944   }
 945 
 946   int null_check_here = code_offset();
 947   switch (type) {
 948     case T_FLOAT: {
 949       if (src->is_single_xmm()) {
 950         __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
 951       } else {
 952         assert(src->is_single_fpu(), "must be");
 953         assert(src->fpu_regnr() == 0, "argument must be on TOS");
 954         if (pop_fpu_stack)      __ fstp_s(as_Address(to_addr));
 955         else                    __ fst_s (as_Address(to_addr));
 956       }
 957       break;
 958     }
 959 
 960     case T_DOUBLE: {
 961       if (src->is_double_xmm()) {
 962         __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
 963       } else {
 964         assert(src->is_double_fpu(), "must be");
 965         assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
 966         if (pop_fpu_stack)      __ fstp_d(as_Address(to_addr));
 967         else                    __ fst_d (as_Address(to_addr));
 968       }
 969       break;
 970     }
 971 
 972     case T_ARRAY:   // fall through
 973     case T_OBJECT:  // fall through
 974       if (UseCompressedOops && !wide) {
 975         __ movl(as_Address(to_addr), compressed_src);
 976       } else {
 977         __ movptr(as_Address(to_addr), src->as_register());
 978       }
 979       break;
 980     case T_METADATA:
 981       // We get here to store a method pointer to the stack to pass to
 982       // a dtrace runtime call. This can't work on 64 bit with
 983       // compressed klass ptrs: T_METADATA can be a compressed klass
 984       // ptr or a 64 bit method pointer.
 985       LP64_ONLY(ShouldNotReachHere());
 986       __ movptr(as_Address(to_addr), src->as_register());
 987       break;
 988     case T_ADDRESS:
 989       __ movptr(as_Address(to_addr), src->as_register());
 990       break;
 991     case T_INT:
 992       __ movl(as_Address(to_addr), src->as_register());
 993       break;
 994 
 995     case T_LONG: {
 996       Register from_lo = src->as_register_lo();
 997       Register from_hi = src->as_register_hi();
 998 #ifdef _LP64
 999       __ movptr(as_Address_lo(to_addr), from_lo);
1000 #else
1001       Register base = to_addr->base()->as_register();
1002       Register index = noreg;
1003       if (to_addr->index()->is_register()) {
1004         index = to_addr->index()->as_register();
1005       }
1006       if (base == from_lo || index == from_lo) {
1007         assert(base != from_hi, "can't be");
1008         assert(index == noreg || (index != base && index != from_hi), "can't handle this");
1009         __ movl(as_Address_hi(to_addr), from_hi);
1010         if (patch != NULL) {
1011           patching_epilog(patch, lir_patch_high, base, info);
1012           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1013           patch_code = lir_patch_low;
1014         }
1015         __ movl(as_Address_lo(to_addr), from_lo);
1016       } else {
1017         assert(index == noreg || (index != base && index != from_lo), "can't handle this");
1018         __ movl(as_Address_lo(to_addr), from_lo);
1019         if (patch != NULL) {
1020           patching_epilog(patch, lir_patch_low, base, info);
1021           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1022           patch_code = lir_patch_high;
1023         }
1024         __ movl(as_Address_hi(to_addr), from_hi);
1025       }
1026 #endif // _LP64
1027       break;
1028     }
1029 
1030     case T_BYTE:    // fall through
1031     case T_BOOLEAN: {
1032       Register src_reg = src->as_register();
1033       Address dst_addr = as_Address(to_addr);
1034       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
1035       __ movb(dst_addr, src_reg);
1036       break;
1037     }
1038 
1039     case T_CHAR:    // fall through
1040     case T_SHORT:
1041       __ movw(as_Address(to_addr), src->as_register());
1042       break;
1043 
1044     default:
1045       ShouldNotReachHere();
1046   }
1047   if (info != NULL) {
1048     add_debug_info_for_null_check(null_check_here, info);
1049   }
1050 
1051   if (patch_code != lir_patch_none) {
1052     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1053   }
1054 }
1055 
1056 
1057 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1058   assert(src->is_stack(), "should not call otherwise");
1059   assert(dest->is_register(), "should not call otherwise");
1060 
1061   if (dest->is_single_cpu()) {
1062     if (type == T_ARRAY || type == T_OBJECT) {
1063       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1064       __ verify_oop(dest->as_register());
1065     } else if (type == T_METADATA) {
1066       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1067     } else {
1068       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1069     }
1070 
1071   } else if (dest->is_double_cpu()) {
1072     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1073     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1074     __ movptr(dest->as_register_lo(), src_addr_LO);
1075     NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1076 
1077   } else if (dest->is_single_xmm()) {
1078     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1079     __ movflt(dest->as_xmm_float_reg(), src_addr);
1080 
1081   } else if (dest->is_double_xmm()) {
1082     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1083     __ movdbl(dest->as_xmm_double_reg(), src_addr);
1084 
1085   } else if (dest->is_single_fpu()) {
1086     assert(dest->fpu_regnr() == 0, "dest must be TOS");
1087     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1088     __ fld_s(src_addr);
1089 
1090   } else if (dest->is_double_fpu()) {
1091     assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1092     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1093     __ fld_d(src_addr);
1094 
1095   } else {
1096     ShouldNotReachHere();
1097   }
1098 }
1099 
1100 
1101 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1102   if (src->is_single_stack()) {
1103     if (type == T_OBJECT || type == T_ARRAY) {
1104       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1105       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1106     } else {
1107 #ifndef _LP64
1108       __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1109       __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1110 #else
1111       //no pushl on 64bits
1112       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1113       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1114 #endif
1115     }
1116 
1117   } else if (src->is_double_stack()) {
1118 #ifdef _LP64
1119     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1120     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1121 #else
1122     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1123     // push and pop the part at src + wordSize, adding wordSize for the previous push
1124     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1125     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1126     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1127 #endif // _LP64
1128 
1129   } else {
1130     ShouldNotReachHere();
1131   }
1132 }
1133 
1134 
1135 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
1136   assert(src->is_address(), "should not call otherwise");
1137   assert(dest->is_register(), "should not call otherwise");
1138 
1139   LIR_Address* addr = src->as_address_ptr();
1140   Address from_addr = as_Address(addr);
1141 
1142   if (addr->base()->type() == T_OBJECT) {
1143     __ verify_oop(addr->base()->as_pointer_register());
1144   }
1145 
1146   switch (type) {
1147     case T_BOOLEAN: // fall through
1148     case T_BYTE:    // fall through
1149     case T_CHAR:    // fall through
1150     case T_SHORT:
1151       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1152         // on pre P6 processors we may get partial register stalls
1153         // so blow away the value of to_rinfo before loading a
1154         // partial word into it.  Do it here so that it precedes
1155         // the potential patch point below.
1156         __ xorptr(dest->as_register(), dest->as_register());
1157       }
1158       break;
1159   }
1160 
1161   PatchingStub* patch = NULL;
1162   if (patch_code != lir_patch_none) {
1163     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1164     assert(from_addr.disp() != 0, "must have");
1165   }
1166   if (info != NULL) {
1167     add_debug_info_for_null_check_here(info);
1168   }
1169 
1170   switch (type) {
1171     case T_FLOAT: {
1172       if (dest->is_single_xmm()) {
1173         __ movflt(dest->as_xmm_float_reg(), from_addr);
1174       } else {
1175         assert(dest->is_single_fpu(), "must be");
1176         assert(dest->fpu_regnr() == 0, "dest must be TOS");
1177         __ fld_s(from_addr);
1178       }
1179       break;
1180     }
1181 
1182     case T_DOUBLE: {
1183       if (dest->is_double_xmm()) {
1184         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1185       } else {
1186         assert(dest->is_double_fpu(), "must be");
1187         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1188         __ fld_d(from_addr);
1189       }
1190       break;
1191     }
1192 
1193     case T_OBJECT:  // fall through
1194     case T_ARRAY:   // fall through
1195       if (UseCompressedOops && !wide) {
1196         __ movl(dest->as_register(), from_addr);
1197       } else {
1198         __ movptr(dest->as_register(), from_addr);
1199       }
1200       break;
1201 
1202     case T_ADDRESS:
1203       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1204         __ movl(dest->as_register(), from_addr);
1205       } else {
1206         __ movptr(dest->as_register(), from_addr);
1207       }
1208       break;
1209     case T_INT:
1210       __ movl(dest->as_register(), from_addr);
1211       break;
1212 
1213     case T_LONG: {
1214       Register to_lo = dest->as_register_lo();
1215       Register to_hi = dest->as_register_hi();
1216 #ifdef _LP64
1217       __ movptr(to_lo, as_Address_lo(addr));
1218 #else
1219       Register base = addr->base()->as_register();
1220       Register index = noreg;
1221       if (addr->index()->is_register()) {
1222         index = addr->index()->as_register();
1223       }
1224       if ((base == to_lo && index == to_hi) ||
1225           (base == to_hi && index == to_lo)) {
1226         // addresses with 2 registers are only formed as a result of
1227         // array access so this code will never have to deal with
1228         // patches or null checks.
1229         assert(info == NULL && patch == NULL, "must be");
1230         __ lea(to_hi, as_Address(addr));
1231         __ movl(to_lo, Address(to_hi, 0));
1232         __ movl(to_hi, Address(to_hi, BytesPerWord));
1233       } else if (base == to_lo || index == to_lo) {
1234         assert(base != to_hi, "can't be");
1235         assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1236         __ movl(to_hi, as_Address_hi(addr));
1237         if (patch != NULL) {
1238           patching_epilog(patch, lir_patch_high, base, info);
1239           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1240           patch_code = lir_patch_low;
1241         }
1242         __ movl(to_lo, as_Address_lo(addr));
1243       } else {
1244         assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1245         __ movl(to_lo, as_Address_lo(addr));
1246         if (patch != NULL) {
1247           patching_epilog(patch, lir_patch_low, base, info);
1248           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1249           patch_code = lir_patch_high;
1250         }
1251         __ movl(to_hi, as_Address_hi(addr));
1252       }
1253 #endif // _LP64
1254       break;
1255     }
1256 
1257     case T_BOOLEAN: // fall through
1258     case T_BYTE: {
1259       Register dest_reg = dest->as_register();
1260       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1261       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1262         __ movsbl(dest_reg, from_addr);
1263       } else {
1264         __ movb(dest_reg, from_addr);
1265         __ shll(dest_reg, 24);
1266         __ sarl(dest_reg, 24);
1267       }
1268       break;
1269     }
1270 
1271     case T_CHAR: {
1272       Register dest_reg = dest->as_register();
1273       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1274       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1275         __ movzwl(dest_reg, from_addr);
1276       } else {
1277         __ movw(dest_reg, from_addr);
1278       }
1279       break;
1280     }
1281 
1282     case T_SHORT: {
1283       Register dest_reg = dest->as_register();
1284       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1285         __ movswl(dest_reg, from_addr);
1286       } else {
1287         __ movw(dest_reg, from_addr);
1288         __ shll(dest_reg, 16);
1289         __ sarl(dest_reg, 16);
1290       }
1291       break;
1292     }
1293 
1294     default:
1295       ShouldNotReachHere();
1296   }
1297 
1298   if (patch != NULL) {
1299     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1300   }
1301 
1302   if (type == T_ARRAY || type == T_OBJECT) {
1303 #ifdef _LP64
1304     if (UseCompressedOops && !wide) {
1305       __ decode_heap_oop(dest->as_register());
1306     }
1307 #endif
1308     __ verify_oop(dest->as_register());
1309   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1310 #ifdef _LP64
1311     if (UseCompressedClassPointers) {
1312       __ decode_klass_not_null(dest->as_register());
1313     }
1314 #endif
1315   }
1316 }
1317 
1318 
1319 NEEDS_CLEANUP; // This could be static?
1320 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1321   int elem_size = type2aelembytes(type);
1322   switch (elem_size) {
1323     case 1: return Address::times_1;
1324     case 2: return Address::times_2;
1325     case 4: return Address::times_4;
1326     case 8: return Address::times_8;
1327   }
1328   ShouldNotReachHere();
1329   return Address::no_scale;
1330 }
1331 
1332 
1333 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1334   switch (op->code()) {
1335     case lir_idiv:
1336     case lir_irem:
1337       arithmetic_idiv(op->code(),
1338                       op->in_opr1(),
1339                       op->in_opr2(),
1340                       op->in_opr3(),
1341                       op->result_opr(),
1342                       op->info());
1343       break;
1344     default:      ShouldNotReachHere(); break;
1345   }
1346 }
1347 
1348 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1349 #ifdef ASSERT
1350   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1351   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1352   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1353 #endif
1354 
1355   if (op->cond() == lir_cond_always) {
1356     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1357     __ jmp (*(op->label()));
1358   } else {
1359     Assembler::Condition acond = Assembler::zero;
1360     if (op->code() == lir_cond_float_branch) {
1361       assert(op->ublock() != NULL, "must have unordered successor");
1362       __ jcc(Assembler::parity, *(op->ublock()->label()));
1363       switch(op->cond()) {
1364         case lir_cond_equal:        acond = Assembler::equal;      break;
1365         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1366         case lir_cond_less:         acond = Assembler::below;      break;
1367         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1368         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1369         case lir_cond_greater:      acond = Assembler::above;      break;
1370         default:                         ShouldNotReachHere();
1371       }
1372     } else {
1373       switch (op->cond()) {
1374         case lir_cond_equal:        acond = Assembler::equal;       break;
1375         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1376         case lir_cond_less:         acond = Assembler::less;        break;
1377         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1378         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1379         case lir_cond_greater:      acond = Assembler::greater;     break;
1380         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1381         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1382         default:                         ShouldNotReachHere();
1383       }
1384     }
1385     __ jcc(acond,*(op->label()));
1386   }
1387 }
1388 
1389 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1390   LIR_Opr src  = op->in_opr();
1391   LIR_Opr dest = op->result_opr();
1392 
1393   switch (op->bytecode()) {
1394     case Bytecodes::_i2l:
1395 #ifdef _LP64
1396       __ movl2ptr(dest->as_register_lo(), src->as_register());
1397 #else
1398       move_regs(src->as_register(), dest->as_register_lo());
1399       move_regs(src->as_register(), dest->as_register_hi());
1400       __ sarl(dest->as_register_hi(), 31);
1401 #endif // LP64
1402       break;
1403 
1404     case Bytecodes::_l2i:
1405 #ifdef _LP64
1406       __ movl(dest->as_register(), src->as_register_lo());
1407 #else
1408       move_regs(src->as_register_lo(), dest->as_register());
1409 #endif
1410       break;
1411 
1412     case Bytecodes::_i2b:
1413       move_regs(src->as_register(), dest->as_register());
1414       __ sign_extend_byte(dest->as_register());
1415       break;
1416 
1417     case Bytecodes::_i2c:
1418       move_regs(src->as_register(), dest->as_register());
1419       __ andl(dest->as_register(), 0xFFFF);
1420       break;
1421 
1422     case Bytecodes::_i2s:
1423       move_regs(src->as_register(), dest->as_register());
1424       __ sign_extend_short(dest->as_register());
1425       break;
1426 
1427 
1428     case Bytecodes::_f2d:
1429     case Bytecodes::_d2f:
1430       if (dest->is_single_xmm()) {
1431         __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1432       } else if (dest->is_double_xmm()) {
1433         __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1434       } else {
1435         assert(src->fpu() == dest->fpu(), "register must be equal");
1436         // do nothing (float result is rounded later through spilling)
1437       }
1438       break;
1439 
1440     case Bytecodes::_i2f:
1441     case Bytecodes::_i2d:
1442       if (dest->is_single_xmm()) {
1443         __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1444       } else if (dest->is_double_xmm()) {
1445         __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1446       } else {
1447         assert(dest->fpu() == 0, "result must be on TOS");
1448         __ movl(Address(rsp, 0), src->as_register());
1449         __ fild_s(Address(rsp, 0));
1450       }
1451       break;
1452 
1453     case Bytecodes::_f2i:
1454     case Bytecodes::_d2i:
1455       if (src->is_single_xmm()) {
1456         __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1457       } else if (src->is_double_xmm()) {
1458         __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1459       } else {
1460         assert(src->fpu() == 0, "input must be on TOS");
1461         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
1462         __ fist_s(Address(rsp, 0));
1463         __ movl(dest->as_register(), Address(rsp, 0));
1464         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1465       }
1466 
1467       // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
1468       assert(op->stub() != NULL, "stub required");
1469       __ cmpl(dest->as_register(), 0x80000000);
1470       __ jcc(Assembler::equal, *op->stub()->entry());
1471       __ bind(*op->stub()->continuation());
1472       break;
1473 
1474     case Bytecodes::_l2f:
1475     case Bytecodes::_l2d:
1476       assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)");
1477       assert(dest->fpu() == 0, "result must be on TOS");
1478 
1479       __ movptr(Address(rsp, 0),            src->as_register_lo());
1480       NOT_LP64(__ movl(Address(rsp, BytesPerWord), src->as_register_hi()));
1481       __ fild_d(Address(rsp, 0));
1482       // float result is rounded later through spilling
1483       break;
1484 
1485     case Bytecodes::_f2l:
1486     case Bytecodes::_d2l:
1487       assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)");
1488       assert(src->fpu() == 0, "input must be on TOS");
1489       assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers");
1490 
1491       // instruction sequence too long to inline it here
1492       {
1493         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1494       }
1495       break;
1496 
1497     default: ShouldNotReachHere();
1498   }
1499 }
1500 
1501 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1502   if (op->init_check()) {
1503     __ cmpb(Address(op->klass()->as_register(),
1504                     InstanceKlass::init_state_offset()),
1505                     InstanceKlass::fully_initialized);
1506     add_debug_info_for_null_check_here(op->stub()->info());
1507     __ jcc(Assembler::notEqual, *op->stub()->entry());
1508   }
1509   __ allocate_object(op->obj()->as_register(),
1510                      op->tmp1()->as_register(),
1511                      op->tmp2()->as_register(),
1512                      op->header_size(),
1513                      op->object_size(),
1514                      op->klass()->as_register(),
1515                      *op->stub()->entry());
1516   __ bind(*op->stub()->continuation());
1517 }
1518 
1519 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1520   Register len =  op->len()->as_register();
1521   LP64_ONLY( __ movslq(len, len); )
1522 
1523   if (UseSlowPath ||
1524       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1525       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1526     __ jmp(*op->stub()->entry());
1527   } else {
1528     Register tmp1 = op->tmp1()->as_register();
1529     Register tmp2 = op->tmp2()->as_register();
1530     Register tmp3 = op->tmp3()->as_register();
1531     if (len == tmp1) {
1532       tmp1 = tmp3;
1533     } else if (len == tmp2) {
1534       tmp2 = tmp3;
1535     } else if (len == tmp3) {
1536       // everything is ok
1537     } else {
1538       __ mov(tmp3, len);
1539     }
1540     __ allocate_array(op->obj()->as_register(),
1541                       len,
1542                       tmp1,
1543                       tmp2,
1544                       arrayOopDesc::header_size(op->type()),
1545                       array_element_size(op->type()),
1546                       op->klass()->as_register(),
1547                       *op->stub()->entry());
1548   }
1549   __ bind(*op->stub()->continuation());
1550 }
1551 
1552 void LIR_Assembler::type_profile_helper(Register mdo,
1553                                         ciMethodData *md, ciProfileData *data,
1554                                         Register recv, Label* update_done) {
1555   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1556     Label next_test;
1557     // See if the receiver is receiver[n].
1558     __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1559     __ jccb(Assembler::notEqual, next_test);
1560     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1561     __ addptr(data_addr, DataLayout::counter_increment);
1562     __ jmp(*update_done);
1563     __ bind(next_test);
1564   }
1565 
1566   // Didn't find receiver; find next empty slot and fill it in
1567   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1568     Label next_test;
1569     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1570     __ cmpptr(recv_addr, (intptr_t)NULL_WORD);
1571     __ jccb(Assembler::notEqual, next_test);
1572     __ movptr(recv_addr, recv);
1573     __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1574     __ jmp(*update_done);
1575     __ bind(next_test);
1576   }
1577 }
1578 
1579 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1580   // we always need a stub for the failure case.
1581   CodeStub* stub = op->stub();
1582   Register obj = op->object()->as_register();
1583   Register k_RInfo = op->tmp1()->as_register();
1584   Register klass_RInfo = op->tmp2()->as_register();
1585   Register dst = op->result_opr()->as_register();
1586   ciKlass* k = op->klass();
1587   Register Rtmp1 = noreg;
1588 
1589   // check if it needs to be profiled
1590   ciMethodData* md = NULL;
1591   ciProfileData* data = NULL;
1592 
1593   if (op->should_profile()) {
1594     ciMethod* method = op->profiled_method();
1595     assert(method != NULL, "Should have method");
1596     int bci = op->profiled_bci();
1597     md = method->method_data_or_null();
1598     assert(md != NULL, "Sanity");
1599     data = md->bci_to_data(bci);
1600     assert(data != NULL,                "need data for type check");
1601     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1602   }
1603   Label profile_cast_success, profile_cast_failure;
1604   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1605   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1606 
1607   if (obj == k_RInfo) {
1608     k_RInfo = dst;
1609   } else if (obj == klass_RInfo) {
1610     klass_RInfo = dst;
1611   }
1612   if (k->is_loaded() && !UseCompressedClassPointers) {
1613     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1614   } else {
1615     Rtmp1 = op->tmp3()->as_register();
1616     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1617   }
1618 
1619   assert_different_registers(obj, k_RInfo, klass_RInfo);
1620 
1621   __ cmpptr(obj, (int32_t)NULL_WORD);
1622   if (op->should_profile()) {
1623     Label not_null;
1624     __ jccb(Assembler::notEqual, not_null);
1625     // Object is null; update MDO and exit
1626     Register mdo  = klass_RInfo;
1627     __ mov_metadata(mdo, md->constant_encoding());
1628     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
1629     int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1630     __ orl(data_addr, header_bits);
1631     __ jmp(*obj_is_null);
1632     __ bind(not_null);
1633   } else {
1634     __ jcc(Assembler::equal, *obj_is_null);
1635   }
1636 
1637   if (!k->is_loaded()) {
1638     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1639   } else {
1640 #ifdef _LP64
1641     __ mov_metadata(k_RInfo, k->constant_encoding());
1642 #endif // _LP64
1643   }
1644   __ verify_oop(obj);
1645 
1646   if (op->fast_check()) {
1647     // get object class
1648     // not a safepoint as obj null check happens earlier
1649 #ifdef _LP64
1650     if (UseCompressedClassPointers) {
1651       __ load_klass(Rtmp1, obj);
1652       __ cmpptr(k_RInfo, Rtmp1);
1653     } else {
1654       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1655     }
1656 #else
1657     if (k->is_loaded()) {
1658       __ cmpklass(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1659     } else {
1660       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1661     }
1662 #endif
1663     __ jcc(Assembler::notEqual, *failure_target);
1664     // successful cast, fall through to profile or jump
1665   } else {
1666     // get object class
1667     // not a safepoint as obj null check happens earlier
1668     __ load_klass(klass_RInfo, obj);
1669     if (k->is_loaded()) {
1670       // See if we get an immediate positive hit
1671 #ifdef _LP64
1672       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1673 #else
1674       __ cmpklass(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1675 #endif // _LP64
1676       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1677         __ jcc(Assembler::notEqual, *failure_target);
1678         // successful cast, fall through to profile or jump
1679       } else {
1680         // See if we get an immediate positive hit
1681         __ jcc(Assembler::equal, *success_target);
1682         // check for self
1683 #ifdef _LP64
1684         __ cmpptr(klass_RInfo, k_RInfo);
1685 #else
1686         __ cmpklass(klass_RInfo, k->constant_encoding());
1687 #endif // _LP64
1688         __ jcc(Assembler::equal, *success_target);
1689 
1690         __ push(klass_RInfo);
1691 #ifdef _LP64
1692         __ push(k_RInfo);
1693 #else
1694         __ pushklass(k->constant_encoding());
1695 #endif // _LP64
1696         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1697         __ pop(klass_RInfo);
1698         __ pop(klass_RInfo);
1699         // result is a boolean
1700         __ cmpl(klass_RInfo, 0);
1701         __ jcc(Assembler::equal, *failure_target);
1702         // successful cast, fall through to profile or jump
1703       }
1704     } else {
1705       // perform the fast part of the checking logic
1706       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1707       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1708       __ push(klass_RInfo);
1709       __ push(k_RInfo);
1710       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1711       __ pop(klass_RInfo);
1712       __ pop(k_RInfo);
1713       // result is a boolean
1714       __ cmpl(k_RInfo, 0);
1715       __ jcc(Assembler::equal, *failure_target);
1716       // successful cast, fall through to profile or jump
1717     }
1718   }
1719   if (op->should_profile()) {
1720     Register mdo  = klass_RInfo, recv = k_RInfo;
1721     __ bind(profile_cast_success);
1722     __ mov_metadata(mdo, md->constant_encoding());
1723     __ load_klass(recv, obj);
1724     Label update_done;
1725     type_profile_helper(mdo, md, data, recv, success);
1726     __ jmp(*success);
1727 
1728     __ bind(profile_cast_failure);
1729     __ mov_metadata(mdo, md->constant_encoding());
1730     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1731     __ subptr(counter_addr, DataLayout::counter_increment);
1732     __ jmp(*failure);
1733   }
1734   __ jmp(*success);
1735 }
1736 
1737 
1738 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1739   LIR_Code code = op->code();
1740   if (code == lir_store_check) {
1741     Register value = op->object()->as_register();
1742     Register array = op->array()->as_register();
1743     Register k_RInfo = op->tmp1()->as_register();
1744     Register klass_RInfo = op->tmp2()->as_register();
1745     Register Rtmp1 = op->tmp3()->as_register();
1746 
1747     CodeStub* stub = op->stub();
1748 
1749     // check if it needs to be profiled
1750     ciMethodData* md = NULL;
1751     ciProfileData* data = NULL;
1752 
1753     if (op->should_profile()) {
1754       ciMethod* method = op->profiled_method();
1755       assert(method != NULL, "Should have method");
1756       int bci = op->profiled_bci();
1757       md = method->method_data_or_null();
1758       assert(md != NULL, "Sanity");
1759       data = md->bci_to_data(bci);
1760       assert(data != NULL,                "need data for type check");
1761       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1762     }
1763     Label profile_cast_success, profile_cast_failure, done;
1764     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1765     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1766 
1767     __ cmpptr(value, (int32_t)NULL_WORD);
1768     if (op->should_profile()) {
1769       Label not_null;
1770       __ jccb(Assembler::notEqual, not_null);
1771       // Object is null; update MDO and exit
1772       Register mdo  = klass_RInfo;
1773       __ mov_metadata(mdo, md->constant_encoding());
1774       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
1775       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1776       __ orl(data_addr, header_bits);
1777       __ jmp(done);
1778       __ bind(not_null);
1779     } else {
1780       __ jcc(Assembler::equal, done);
1781     }
1782 
1783     add_debug_info_for_null_check_here(op->info_for_exception());
1784     __ load_klass(k_RInfo, array);
1785     __ load_klass(klass_RInfo, value);
1786 
1787     // get instance klass (it's already uncompressed)
1788     __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1789     // perform the fast part of the checking logic
1790     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1791     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1792     __ push(klass_RInfo);
1793     __ push(k_RInfo);
1794     __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1795     __ pop(klass_RInfo);
1796     __ pop(k_RInfo);
1797     // result is a boolean
1798     __ cmpl(k_RInfo, 0);
1799     __ jcc(Assembler::equal, *failure_target);
1800     // fall through to the success case
1801 
1802     if (op->should_profile()) {
1803       Register mdo  = klass_RInfo, recv = k_RInfo;
1804       __ bind(profile_cast_success);
1805       __ mov_metadata(mdo, md->constant_encoding());
1806       __ load_klass(recv, value);
1807       Label update_done;
1808       type_profile_helper(mdo, md, data, recv, &done);
1809       __ jmpb(done);
1810 
1811       __ bind(profile_cast_failure);
1812       __ mov_metadata(mdo, md->constant_encoding());
1813       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1814       __ subptr(counter_addr, DataLayout::counter_increment);
1815       __ jmp(*stub->entry());
1816     }
1817 
1818     __ bind(done);
1819   } else
1820     if (code == lir_checkcast) {
1821       Register obj = op->object()->as_register();
1822       Register dst = op->result_opr()->as_register();
1823       Label success;
1824       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1825       __ bind(success);
1826       if (dst != obj) {
1827         __ mov(dst, obj);
1828       }
1829     } else
1830       if (code == lir_instanceof) {
1831         Register obj = op->object()->as_register();
1832         Register dst = op->result_opr()->as_register();
1833         Label success, failure, done;
1834         emit_typecheck_helper(op, &success, &failure, &failure);
1835         __ bind(failure);
1836         __ xorptr(dst, dst);
1837         __ jmpb(done);
1838         __ bind(success);
1839         __ movptr(dst, 1);
1840         __ bind(done);
1841       } else {
1842         ShouldNotReachHere();
1843       }
1844 
1845 }
1846 
1847 
1848 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1849   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
1850     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1851     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1852     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1853     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1854     Register addr = op->addr()->as_register();
1855     if (os::is_MP()) {
1856       __ lock();
1857     }
1858     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1859 
1860   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1861     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1862     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1863     Register newval = op->new_value()->as_register();
1864     Register cmpval = op->cmp_value()->as_register();
1865     assert(cmpval == rax, "wrong register");
1866     assert(newval != NULL, "new val must be register");
1867     assert(cmpval != newval, "cmp and new values must be in different registers");
1868     assert(cmpval != addr, "cmp and addr must be in different registers");
1869     assert(newval != addr, "new value and addr must be in different registers");
1870 
1871     if ( op->code() == lir_cas_obj) {
1872 #ifdef _LP64
1873       if (UseCompressedOops) {
1874         __ encode_heap_oop(cmpval);
1875         __ mov(rscratch1, newval);
1876         __ encode_heap_oop(rscratch1);
1877         if (os::is_MP()) {
1878           __ lock();
1879         }
1880         // cmpval (rax) is implicitly used by this instruction
1881         __ cmpxchgl(rscratch1, Address(addr, 0));
1882       } else
1883 #endif
1884       {
1885         if (os::is_MP()) {
1886           __ lock();
1887         }
1888         __ cmpxchgptr(newval, Address(addr, 0));
1889       }
1890     } else {
1891       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1892       if (os::is_MP()) {
1893         __ lock();
1894       }
1895       __ cmpxchgl(newval, Address(addr, 0));
1896     }
1897 #ifdef _LP64
1898   } else if (op->code() == lir_cas_long) {
1899     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1900     Register newval = op->new_value()->as_register_lo();
1901     Register cmpval = op->cmp_value()->as_register_lo();
1902     assert(cmpval == rax, "wrong register");
1903     assert(newval != NULL, "new val must be register");
1904     assert(cmpval != newval, "cmp and new values must be in different registers");
1905     assert(cmpval != addr, "cmp and addr must be in different registers");
1906     assert(newval != addr, "new value and addr must be in different registers");
1907     if (os::is_MP()) {
1908       __ lock();
1909     }
1910     __ cmpxchgq(newval, Address(addr, 0));
1911 #endif // _LP64
1912   } else {
1913     Unimplemented();
1914   }
1915 }
1916 
1917 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1918   Assembler::Condition acond, ncond;
1919   switch (condition) {
1920     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1921     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1922     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1923     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1924     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1925     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1926     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1927     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1928     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1929                                 ShouldNotReachHere();
1930   }
1931 
1932   if (opr1->is_cpu_register()) {
1933     reg2reg(opr1, result);
1934   } else if (opr1->is_stack()) {
1935     stack2reg(opr1, result, result->type());
1936   } else if (opr1->is_constant()) {
1937     const2reg(opr1, result, lir_patch_none, NULL);
1938   } else {
1939     ShouldNotReachHere();
1940   }
1941 
1942   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1943     // optimized version that does not require a branch
1944     if (opr2->is_single_cpu()) {
1945       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1946       __ cmov(ncond, result->as_register(), opr2->as_register());
1947     } else if (opr2->is_double_cpu()) {
1948       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1949       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1950       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1951       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
1952     } else if (opr2->is_single_stack()) {
1953       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1954     } else if (opr2->is_double_stack()) {
1955       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1956       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
1957     } else {
1958       ShouldNotReachHere();
1959     }
1960 
1961   } else {
1962     Label skip;
1963     __ jcc (acond, skip);
1964     if (opr2->is_cpu_register()) {
1965       reg2reg(opr2, result);
1966     } else if (opr2->is_stack()) {
1967       stack2reg(opr2, result, result->type());
1968     } else if (opr2->is_constant()) {
1969       const2reg(opr2, result, lir_patch_none, NULL);
1970     } else {
1971       ShouldNotReachHere();
1972     }
1973     __ bind(skip);
1974   }
1975 }
1976 
1977 
1978 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1979   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1980 
1981   if (left->is_single_cpu()) {
1982     assert(left == dest, "left and dest must be equal");
1983     Register lreg = left->as_register();
1984 
1985     if (right->is_single_cpu()) {
1986       // cpu register - cpu register
1987       Register rreg = right->as_register();
1988       switch (code) {
1989         case lir_add: __ addl (lreg, rreg); break;
1990         case lir_sub: __ subl (lreg, rreg); break;
1991         case lir_mul: __ imull(lreg, rreg); break;
1992         default:      ShouldNotReachHere();
1993       }
1994 
1995     } else if (right->is_stack()) {
1996       // cpu register - stack
1997       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1998       switch (code) {
1999         case lir_add: __ addl(lreg, raddr); break;
2000         case lir_sub: __ subl(lreg, raddr); break;
2001         default:      ShouldNotReachHere();
2002       }
2003 
2004     } else if (right->is_constant()) {
2005       // cpu register - constant
2006       jint c = right->as_constant_ptr()->as_jint();
2007       switch (code) {
2008         case lir_add: {
2009           __ incrementl(lreg, c);
2010           break;
2011         }
2012         case lir_sub: {
2013           __ decrementl(lreg, c);
2014           break;
2015         }
2016         default: ShouldNotReachHere();
2017       }
2018 
2019     } else {
2020       ShouldNotReachHere();
2021     }
2022 
2023   } else if (left->is_double_cpu()) {
2024     assert(left == dest, "left and dest must be equal");
2025     Register lreg_lo = left->as_register_lo();
2026     Register lreg_hi = left->as_register_hi();
2027 
2028     if (right->is_double_cpu()) {
2029       // cpu register - cpu register
2030       Register rreg_lo = right->as_register_lo();
2031       Register rreg_hi = right->as_register_hi();
2032       NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
2033       LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
2034       switch (code) {
2035         case lir_add:
2036           __ addptr(lreg_lo, rreg_lo);
2037           NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2038           break;
2039         case lir_sub:
2040           __ subptr(lreg_lo, rreg_lo);
2041           NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2042           break;
2043         case lir_mul:
2044 #ifdef _LP64
2045           __ imulq(lreg_lo, rreg_lo);
2046 #else
2047           assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2048           __ imull(lreg_hi, rreg_lo);
2049           __ imull(rreg_hi, lreg_lo);
2050           __ addl (rreg_hi, lreg_hi);
2051           __ mull (rreg_lo);
2052           __ addl (lreg_hi, rreg_hi);
2053 #endif // _LP64
2054           break;
2055         default:
2056           ShouldNotReachHere();
2057       }
2058 
2059     } else if (right->is_constant()) {
2060       // cpu register - constant
2061 #ifdef _LP64
2062       jlong c = right->as_constant_ptr()->as_jlong_bits();
2063       __ movptr(r10, (intptr_t) c);
2064       switch (code) {
2065         case lir_add:
2066           __ addptr(lreg_lo, r10);
2067           break;
2068         case lir_sub:
2069           __ subptr(lreg_lo, r10);
2070           break;
2071         default:
2072           ShouldNotReachHere();
2073       }
2074 #else
2075       jint c_lo = right->as_constant_ptr()->as_jint_lo();
2076       jint c_hi = right->as_constant_ptr()->as_jint_hi();
2077       switch (code) {
2078         case lir_add:
2079           __ addptr(lreg_lo, c_lo);
2080           __ adcl(lreg_hi, c_hi);
2081           break;
2082         case lir_sub:
2083           __ subptr(lreg_lo, c_lo);
2084           __ sbbl(lreg_hi, c_hi);
2085           break;
2086         default:
2087           ShouldNotReachHere();
2088       }
2089 #endif // _LP64
2090 
2091     } else {
2092       ShouldNotReachHere();
2093     }
2094 
2095   } else if (left->is_single_xmm()) {
2096     assert(left == dest, "left and dest must be equal");
2097     XMMRegister lreg = left->as_xmm_float_reg();
2098 
2099     if (right->is_single_xmm()) {
2100       XMMRegister rreg = right->as_xmm_float_reg();
2101       switch (code) {
2102         case lir_add: __ addss(lreg, rreg);  break;
2103         case lir_sub: __ subss(lreg, rreg);  break;
2104         case lir_mul_strictfp: // fall through
2105         case lir_mul: __ mulss(lreg, rreg);  break;
2106         case lir_div_strictfp: // fall through
2107         case lir_div: __ divss(lreg, rreg);  break;
2108         default: ShouldNotReachHere();
2109       }
2110     } else {
2111       Address raddr;
2112       if (right->is_single_stack()) {
2113         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2114       } else if (right->is_constant()) {
2115         // hack for now
2116         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2117       } else {
2118         ShouldNotReachHere();
2119       }
2120       switch (code) {
2121         case lir_add: __ addss(lreg, raddr);  break;
2122         case lir_sub: __ subss(lreg, raddr);  break;
2123         case lir_mul_strictfp: // fall through
2124         case lir_mul: __ mulss(lreg, raddr);  break;
2125         case lir_div_strictfp: // fall through
2126         case lir_div: __ divss(lreg, raddr);  break;
2127         default: ShouldNotReachHere();
2128       }
2129     }
2130 
2131   } else if (left->is_double_xmm()) {
2132     assert(left == dest, "left and dest must be equal");
2133 
2134     XMMRegister lreg = left->as_xmm_double_reg();
2135     if (right->is_double_xmm()) {
2136       XMMRegister rreg = right->as_xmm_double_reg();
2137       switch (code) {
2138         case lir_add: __ addsd(lreg, rreg);  break;
2139         case lir_sub: __ subsd(lreg, rreg);  break;
2140         case lir_mul_strictfp: // fall through
2141         case lir_mul: __ mulsd(lreg, rreg);  break;
2142         case lir_div_strictfp: // fall through
2143         case lir_div: __ divsd(lreg, rreg);  break;
2144         default: ShouldNotReachHere();
2145       }
2146     } else {
2147       Address raddr;
2148       if (right->is_double_stack()) {
2149         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2150       } else if (right->is_constant()) {
2151         // hack for now
2152         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2153       } else {
2154         ShouldNotReachHere();
2155       }
2156       switch (code) {
2157         case lir_add: __ addsd(lreg, raddr);  break;
2158         case lir_sub: __ subsd(lreg, raddr);  break;
2159         case lir_mul_strictfp: // fall through
2160         case lir_mul: __ mulsd(lreg, raddr);  break;
2161         case lir_div_strictfp: // fall through
2162         case lir_div: __ divsd(lreg, raddr);  break;
2163         default: ShouldNotReachHere();
2164       }
2165     }
2166 
2167   } else if (left->is_single_fpu()) {
2168     assert(dest->is_single_fpu(),  "fpu stack allocation required");
2169 
2170     if (right->is_single_fpu()) {
2171       arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack);
2172 
2173     } else {
2174       assert(left->fpu_regnr() == 0, "left must be on TOS");
2175       assert(dest->fpu_regnr() == 0, "dest must be on TOS");
2176 
2177       Address raddr;
2178       if (right->is_single_stack()) {
2179         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2180       } else if (right->is_constant()) {
2181         address const_addr = float_constant(right->as_jfloat());
2182         assert(const_addr != NULL, "incorrect float/double constant maintainance");
2183         // hack for now
2184         raddr = __ as_Address(InternalAddress(const_addr));
2185       } else {
2186         ShouldNotReachHere();
2187       }
2188 
2189       switch (code) {
2190         case lir_add: __ fadd_s(raddr); break;
2191         case lir_sub: __ fsub_s(raddr); break;
2192         case lir_mul_strictfp: // fall through
2193         case lir_mul: __ fmul_s(raddr); break;
2194         case lir_div_strictfp: // fall through
2195         case lir_div: __ fdiv_s(raddr); break;
2196         default:      ShouldNotReachHere();
2197       }
2198     }
2199 
2200   } else if (left->is_double_fpu()) {
2201     assert(dest->is_double_fpu(),  "fpu stack allocation required");
2202 
2203     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2204       // Double values require special handling for strictfp mul/div on x86
2205       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
2206       __ fmulp(left->fpu_regnrLo() + 1);
2207     }
2208 
2209     if (right->is_double_fpu()) {
2210       arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack);
2211 
2212     } else {
2213       assert(left->fpu_regnrLo() == 0, "left must be on TOS");
2214       assert(dest->fpu_regnrLo() == 0, "dest must be on TOS");
2215 
2216       Address raddr;
2217       if (right->is_double_stack()) {
2218         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2219       } else if (right->is_constant()) {
2220         // hack for now
2221         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2222       } else {
2223         ShouldNotReachHere();
2224       }
2225 
2226       switch (code) {
2227         case lir_add: __ fadd_d(raddr); break;
2228         case lir_sub: __ fsub_d(raddr); break;
2229         case lir_mul_strictfp: // fall through
2230         case lir_mul: __ fmul_d(raddr); break;
2231         case lir_div_strictfp: // fall through
2232         case lir_div: __ fdiv_d(raddr); break;
2233         default: ShouldNotReachHere();
2234       }
2235     }
2236 
2237     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2238       // Double values require special handling for strictfp mul/div on x86
2239       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
2240       __ fmulp(dest->fpu_regnrLo() + 1);
2241     }
2242 
2243   } else if (left->is_single_stack() || left->is_address()) {
2244     assert(left == dest, "left and dest must be equal");
2245 
2246     Address laddr;
2247     if (left->is_single_stack()) {
2248       laddr = frame_map()->address_for_slot(left->single_stack_ix());
2249     } else if (left->is_address()) {
2250       laddr = as_Address(left->as_address_ptr());
2251     } else {
2252       ShouldNotReachHere();
2253     }
2254 
2255     if (right->is_single_cpu()) {
2256       Register rreg = right->as_register();
2257       switch (code) {
2258         case lir_add: __ addl(laddr, rreg); break;
2259         case lir_sub: __ subl(laddr, rreg); break;
2260         default:      ShouldNotReachHere();
2261       }
2262     } else if (right->is_constant()) {
2263       jint c = right->as_constant_ptr()->as_jint();
2264       switch (code) {
2265         case lir_add: {
2266           __ incrementl(laddr, c);
2267           break;
2268         }
2269         case lir_sub: {
2270           __ decrementl(laddr, c);
2271           break;
2272         }
2273         default: ShouldNotReachHere();
2274       }
2275     } else {
2276       ShouldNotReachHere();
2277     }
2278 
2279   } else {
2280     ShouldNotReachHere();
2281   }
2282 }
2283 
2284 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) {
2285   assert(pop_fpu_stack  || (left_index     == dest_index || right_index     == dest_index), "invalid LIR");
2286   assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR");
2287   assert(left_index == 0 || right_index == 0, "either must be on top of stack");
2288 
2289   bool left_is_tos = (left_index == 0);
2290   bool dest_is_tos = (dest_index == 0);
2291   int non_tos_index = (left_is_tos ? right_index : left_index);
2292 
2293   switch (code) {
2294     case lir_add:
2295       if (pop_fpu_stack)       __ faddp(non_tos_index);
2296       else if (dest_is_tos)    __ fadd (non_tos_index);
2297       else                     __ fadda(non_tos_index);
2298       break;
2299 
2300     case lir_sub:
2301       if (left_is_tos) {
2302         if (pop_fpu_stack)     __ fsubrp(non_tos_index);
2303         else if (dest_is_tos)  __ fsub  (non_tos_index);
2304         else                   __ fsubra(non_tos_index);
2305       } else {
2306         if (pop_fpu_stack)     __ fsubp (non_tos_index);
2307         else if (dest_is_tos)  __ fsubr (non_tos_index);
2308         else                   __ fsuba (non_tos_index);
2309       }
2310       break;
2311 
2312     case lir_mul_strictfp: // fall through
2313     case lir_mul:
2314       if (pop_fpu_stack)       __ fmulp(non_tos_index);
2315       else if (dest_is_tos)    __ fmul (non_tos_index);
2316       else                     __ fmula(non_tos_index);
2317       break;
2318 
2319     case lir_div_strictfp: // fall through
2320     case lir_div:
2321       if (left_is_tos) {
2322         if (pop_fpu_stack)     __ fdivrp(non_tos_index);
2323         else if (dest_is_tos)  __ fdiv  (non_tos_index);
2324         else                   __ fdivra(non_tos_index);
2325       } else {
2326         if (pop_fpu_stack)     __ fdivp (non_tos_index);
2327         else if (dest_is_tos)  __ fdivr (non_tos_index);
2328         else                   __ fdiva (non_tos_index);
2329       }
2330       break;
2331 
2332     case lir_rem:
2333       assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation");
2334       __ fremr(noreg);
2335       break;
2336 
2337     default:
2338       ShouldNotReachHere();
2339   }
2340 }
2341 
2342 
2343 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
2344   if (value->is_double_xmm()) {
2345     switch(code) {
2346       case lir_abs :
2347         {
2348           if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2349             __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2350           }
2351           __ andpd(dest->as_xmm_double_reg(),
2352                     ExternalAddress((address)double_signmask_pool));
2353         }
2354         break;
2355 
2356       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2357       // all other intrinsics are not available in the SSE instruction set, so FPU is used
2358       default      : ShouldNotReachHere();
2359     }
2360 
2361   } else if (value->is_double_fpu()) {
2362     assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2363     switch(code) {
2364       case lir_log10 : __ flog10() ; break;
2365       case lir_abs   : __ fabs() ; break;
2366       case lir_sqrt  : __ fsqrt(); break;
2367       case lir_sin   :
2368         // Should consider not saving rbx, if not necessary
2369         __ trigfunc('s', op->as_Op2()->fpu_stack_size());
2370         break;
2371       case lir_cos :
2372         // Should consider not saving rbx, if not necessary
2373         assert(op->as_Op2()->fpu_stack_size() <= 6, "sin and cos need two free stack slots");
2374         __ trigfunc('c', op->as_Op2()->fpu_stack_size());
2375         break;
2376       case lir_tan :
2377         // Should consider not saving rbx, if not necessary
2378         __ trigfunc('t', op->as_Op2()->fpu_stack_size());
2379         break;
2380       case lir_pow :
2381         __ pow_with_fallback(op->as_Op2()->fpu_stack_size());
2382         break;
2383       default      : ShouldNotReachHere();
2384     }
2385   } else {
2386     Unimplemented();
2387   }
2388 }
2389 
2390 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2391   // assert(left->destroys_register(), "check");
2392   if (left->is_single_cpu()) {
2393     Register reg = left->as_register();
2394     if (right->is_constant()) {
2395       int val = right->as_constant_ptr()->as_jint();
2396       switch (code) {
2397         case lir_logic_and: __ andl (reg, val); break;
2398         case lir_logic_or:  __ orl  (reg, val); break;
2399         case lir_logic_xor: __ xorl (reg, val); break;
2400         default: ShouldNotReachHere();
2401       }
2402     } else if (right->is_stack()) {
2403       // added support for stack operands
2404       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2405       switch (code) {
2406         case lir_logic_and: __ andl (reg, raddr); break;
2407         case lir_logic_or:  __ orl  (reg, raddr); break;
2408         case lir_logic_xor: __ xorl (reg, raddr); break;
2409         default: ShouldNotReachHere();
2410       }
2411     } else {
2412       Register rright = right->as_register();
2413       switch (code) {
2414         case lir_logic_and: __ andptr (reg, rright); break;
2415         case lir_logic_or : __ orptr  (reg, rright); break;
2416         case lir_logic_xor: __ xorptr (reg, rright); break;
2417         default: ShouldNotReachHere();
2418       }
2419     }
2420     move_regs(reg, dst->as_register());
2421   } else {
2422     Register l_lo = left->as_register_lo();
2423     Register l_hi = left->as_register_hi();
2424     if (right->is_constant()) {
2425 #ifdef _LP64
2426       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2427       switch (code) {
2428         case lir_logic_and:
2429           __ andq(l_lo, rscratch1);
2430           break;
2431         case lir_logic_or:
2432           __ orq(l_lo, rscratch1);
2433           break;
2434         case lir_logic_xor:
2435           __ xorq(l_lo, rscratch1);
2436           break;
2437         default: ShouldNotReachHere();
2438       }
2439 #else
2440       int r_lo = right->as_constant_ptr()->as_jint_lo();
2441       int r_hi = right->as_constant_ptr()->as_jint_hi();
2442       switch (code) {
2443         case lir_logic_and:
2444           __ andl(l_lo, r_lo);
2445           __ andl(l_hi, r_hi);
2446           break;
2447         case lir_logic_or:
2448           __ orl(l_lo, r_lo);
2449           __ orl(l_hi, r_hi);
2450           break;
2451         case lir_logic_xor:
2452           __ xorl(l_lo, r_lo);
2453           __ xorl(l_hi, r_hi);
2454           break;
2455         default: ShouldNotReachHere();
2456       }
2457 #endif // _LP64
2458     } else {
2459 #ifdef _LP64
2460       Register r_lo;
2461       if (right->type() == T_OBJECT || right->type() == T_ARRAY) {
2462         r_lo = right->as_register();
2463       } else {
2464         r_lo = right->as_register_lo();
2465       }
2466 #else
2467       Register r_lo = right->as_register_lo();
2468       Register r_hi = right->as_register_hi();
2469       assert(l_lo != r_hi, "overwriting registers");
2470 #endif
2471       switch (code) {
2472         case lir_logic_and:
2473           __ andptr(l_lo, r_lo);
2474           NOT_LP64(__ andptr(l_hi, r_hi);)
2475           break;
2476         case lir_logic_or:
2477           __ orptr(l_lo, r_lo);
2478           NOT_LP64(__ orptr(l_hi, r_hi);)
2479           break;
2480         case lir_logic_xor:
2481           __ xorptr(l_lo, r_lo);
2482           NOT_LP64(__ xorptr(l_hi, r_hi);)
2483           break;
2484         default: ShouldNotReachHere();
2485       }
2486     }
2487 
2488     Register dst_lo = dst->as_register_lo();
2489     Register dst_hi = dst->as_register_hi();
2490 
2491 #ifdef _LP64
2492     move_regs(l_lo, dst_lo);
2493 #else
2494     if (dst_lo == l_hi) {
2495       assert(dst_hi != l_lo, "overwriting registers");
2496       move_regs(l_hi, dst_hi);
2497       move_regs(l_lo, dst_lo);
2498     } else {
2499       assert(dst_lo != l_hi, "overwriting registers");
2500       move_regs(l_lo, dst_lo);
2501       move_regs(l_hi, dst_hi);
2502     }
2503 #endif // _LP64
2504   }
2505 }
2506 
2507 
2508 // we assume that rax, and rdx can be overwritten
2509 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2510 
2511   assert(left->is_single_cpu(),   "left must be register");
2512   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2513   assert(result->is_single_cpu(), "result must be register");
2514 
2515   //  assert(left->destroys_register(), "check");
2516   //  assert(right->destroys_register(), "check");
2517 
2518   Register lreg = left->as_register();
2519   Register dreg = result->as_register();
2520 
2521   if (right->is_constant()) {
2522     int divisor = right->as_constant_ptr()->as_jint();
2523     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2524     if (code == lir_idiv) {
2525       assert(lreg == rax, "must be rax,");
2526       assert(temp->as_register() == rdx, "tmp register must be rdx");
2527       __ cdql(); // sign extend into rdx:rax
2528       if (divisor == 2) {
2529         __ subl(lreg, rdx);
2530       } else {
2531         __ andl(rdx, divisor - 1);
2532         __ addl(lreg, rdx);
2533       }
2534       __ sarl(lreg, log2_intptr(divisor));
2535       move_regs(lreg, dreg);
2536     } else if (code == lir_irem) {
2537       Label done;
2538       __ mov(dreg, lreg);
2539       __ andl(dreg, 0x80000000 | (divisor - 1));
2540       __ jcc(Assembler::positive, done);
2541       __ decrement(dreg);
2542       __ orl(dreg, ~(divisor - 1));
2543       __ increment(dreg);
2544       __ bind(done);
2545     } else {
2546       ShouldNotReachHere();
2547     }
2548   } else {
2549     Register rreg = right->as_register();
2550     assert(lreg == rax, "left register must be rax,");
2551     assert(rreg != rdx, "right register must not be rdx");
2552     assert(temp->as_register() == rdx, "tmp register must be rdx");
2553 
2554     move_regs(lreg, rax);
2555 
2556     int idivl_offset = __ corrected_idivl(rreg);
2557     add_debug_info_for_div0(idivl_offset, info);
2558     if (code == lir_irem) {
2559       move_regs(rdx, dreg); // result is in rdx
2560     } else {
2561       move_regs(rax, dreg);
2562     }
2563   }
2564 }
2565 
2566 
2567 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2568   if (opr1->is_single_cpu()) {
2569     Register reg1 = opr1->as_register();
2570     if (opr2->is_single_cpu()) {
2571       // cpu register - cpu register
2572       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2573         __ cmpptr(reg1, opr2->as_register());
2574       } else {
2575         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
2576         __ cmpl(reg1, opr2->as_register());
2577       }
2578     } else if (opr2->is_stack()) {
2579       // cpu register - stack
2580       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2581         __ cmpptr(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2582       } else {
2583         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2584       }
2585     } else if (opr2->is_constant()) {
2586       // cpu register - constant
2587       LIR_Const* c = opr2->as_constant_ptr();
2588       if (c->type() == T_INT) {
2589         __ cmpl(reg1, c->as_jint());
2590       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2591         // In 64bit oops are single register
2592         jobject o = c->as_jobject();
2593         if (o == NULL) {
2594           __ cmpptr(reg1, (int32_t)NULL_WORD);
2595         } else {
2596 #ifdef _LP64
2597           __ movoop(rscratch1, o);
2598           __ cmpptr(reg1, rscratch1);
2599 #else
2600           __ cmpoop(reg1, c->as_jobject());
2601 #endif // _LP64
2602         }
2603       } else {
2604         fatal("unexpected type: %s", basictype_to_str(c->type()));
2605       }
2606       // cpu register - address
2607     } else if (opr2->is_address()) {
2608       if (op->info() != NULL) {
2609         add_debug_info_for_null_check_here(op->info());
2610       }
2611       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2612     } else {
2613       ShouldNotReachHere();
2614     }
2615 
2616   } else if(opr1->is_double_cpu()) {
2617     Register xlo = opr1->as_register_lo();
2618     Register xhi = opr1->as_register_hi();
2619     if (opr2->is_double_cpu()) {
2620 #ifdef _LP64
2621       __ cmpptr(xlo, opr2->as_register_lo());
2622 #else
2623       // cpu register - cpu register
2624       Register ylo = opr2->as_register_lo();
2625       Register yhi = opr2->as_register_hi();
2626       __ subl(xlo, ylo);
2627       __ sbbl(xhi, yhi);
2628       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2629         __ orl(xhi, xlo);
2630       }
2631 #endif // _LP64
2632     } else if (opr2->is_constant()) {
2633       // cpu register - constant 0
2634       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2635 #ifdef _LP64
2636       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2637 #else
2638       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2639       __ orl(xhi, xlo);
2640 #endif // _LP64
2641     } else {
2642       ShouldNotReachHere();
2643     }
2644 
2645   } else if (opr1->is_single_xmm()) {
2646     XMMRegister reg1 = opr1->as_xmm_float_reg();
2647     if (opr2->is_single_xmm()) {
2648       // xmm register - xmm register
2649       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2650     } else if (opr2->is_stack()) {
2651       // xmm register - stack
2652       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2653     } else if (opr2->is_constant()) {
2654       // xmm register - constant
2655       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2656     } else if (opr2->is_address()) {
2657       // xmm register - address
2658       if (op->info() != NULL) {
2659         add_debug_info_for_null_check_here(op->info());
2660       }
2661       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2662     } else {
2663       ShouldNotReachHere();
2664     }
2665 
2666   } else if (opr1->is_double_xmm()) {
2667     XMMRegister reg1 = opr1->as_xmm_double_reg();
2668     if (opr2->is_double_xmm()) {
2669       // xmm register - xmm register
2670       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2671     } else if (opr2->is_stack()) {
2672       // xmm register - stack
2673       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2674     } else if (opr2->is_constant()) {
2675       // xmm register - constant
2676       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2677     } else if (opr2->is_address()) {
2678       // xmm register - address
2679       if (op->info() != NULL) {
2680         add_debug_info_for_null_check_here(op->info());
2681       }
2682       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2683     } else {
2684       ShouldNotReachHere();
2685     }
2686 
2687   } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) {
2688     assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)");
2689     assert(opr2->is_fpu_register(), "both must be registers");
2690     __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2691 
2692   } else if (opr1->is_address() && opr2->is_constant()) {
2693     LIR_Const* c = opr2->as_constant_ptr();
2694 #ifdef _LP64
2695     if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2696       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2697       __ movoop(rscratch1, c->as_jobject());
2698     }
2699 #endif // LP64
2700     if (op->info() != NULL) {
2701       add_debug_info_for_null_check_here(op->info());
2702     }
2703     // special case: address - constant
2704     LIR_Address* addr = opr1->as_address_ptr();
2705     if (c->type() == T_INT) {
2706       __ cmpl(as_Address(addr), c->as_jint());
2707     } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2708 #ifdef _LP64
2709       // %%% Make this explode if addr isn't reachable until we figure out a
2710       // better strategy by giving noreg as the temp for as_Address
2711       __ cmpptr(rscratch1, as_Address(addr, noreg));
2712 #else
2713       __ cmpoop(as_Address(addr), c->as_jobject());
2714 #endif // _LP64
2715     } else {
2716       ShouldNotReachHere();
2717     }
2718 
2719   } else {
2720     ShouldNotReachHere();
2721   }
2722 }
2723 
2724 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2725   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2726     if (left->is_single_xmm()) {
2727       assert(right->is_single_xmm(), "must match");
2728       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2729     } else if (left->is_double_xmm()) {
2730       assert(right->is_double_xmm(), "must match");
2731       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2732 
2733     } else {
2734       assert(left->is_single_fpu() || left->is_double_fpu(), "must be");
2735       assert(right->is_single_fpu() || right->is_double_fpu(), "must match");
2736 
2737       assert(left->fpu() == 0, "left must be on TOS");
2738       __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2739                   op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2740     }
2741   } else {
2742     assert(code == lir_cmp_l2i, "check");
2743 #ifdef _LP64
2744     Label done;
2745     Register dest = dst->as_register();
2746     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2747     __ movl(dest, -1);
2748     __ jccb(Assembler::less, done);
2749     __ set_byte_if_not_zero(dest);
2750     __ movzbl(dest, dest);
2751     __ bind(done);
2752 #else
2753     __ lcmp2int(left->as_register_hi(),
2754                 left->as_register_lo(),
2755                 right->as_register_hi(),
2756                 right->as_register_lo());
2757     move_regs(left->as_register_hi(), dst->as_register());
2758 #endif // _LP64
2759   }
2760 }
2761 
2762 
2763 void LIR_Assembler::align_call(LIR_Code code) {
2764   if (os::is_MP()) {
2765     // make sure that the displacement word of the call ends up word aligned
2766     int offset = __ offset();
2767     switch (code) {
2768       case lir_static_call:
2769       case lir_optvirtual_call:
2770       case lir_dynamic_call:
2771         offset += NativeCall::displacement_offset;
2772         break;
2773       case lir_icvirtual_call:
2774         offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
2775       break;
2776       case lir_virtual_call:  // currently, sparc-specific for niagara
2777       default: ShouldNotReachHere();
2778     }
2779     __ align(BytesPerWord, offset);
2780   }
2781 }
2782 
2783 
2784 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2785   assert(!os::is_MP() || (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2786          "must be aligned");
2787   __ call(AddressLiteral(op->addr(), rtype));
2788   add_call_info(code_offset(), op->info());
2789 }
2790 
2791 
2792 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2793   __ ic_call(op->addr());
2794   add_call_info(code_offset(), op->info());
2795   assert(!os::is_MP() ||
2796          (__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2797          "must be aligned");
2798 }
2799 
2800 
2801 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2802 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2803   ShouldNotReachHere();
2804 }
2805 
2806 
2807 void LIR_Assembler::emit_static_call_stub() {
2808   address call_pc = __ pc();
2809   address stub = __ start_a_stub(call_stub_size);
2810   if (stub == NULL) {
2811     bailout("static call stub overflow");
2812     return;
2813   }
2814 
2815   int start = __ offset();
2816   if (os::is_MP()) {
2817     // make sure that the displacement word of the call ends up word aligned
2818     __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset);
2819   }
2820   __ relocate(static_stub_Relocation::spec(call_pc));
2821   __ mov_metadata(rbx, (Metadata*)NULL);
2822   // must be set to -1 at code generation time
2823   assert(!os::is_MP() || ((__ offset() + 1) % BytesPerWord) == 0, "must be aligned on MP");
2824   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2825   __ jump(RuntimeAddress(__ pc()));
2826 
2827   assert(__ offset() - start <= call_stub_size, "stub too big");
2828   __ end_a_stub();
2829 }
2830 
2831 
2832 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2833   assert(exceptionOop->as_register() == rax, "must match");
2834   assert(exceptionPC->as_register() == rdx, "must match");
2835 
2836   // exception object is not added to oop map by LinearScan
2837   // (LinearScan assumes that no oops are in fixed registers)
2838   info->add_register_oop(exceptionOop);
2839   Runtime1::StubID unwind_id;
2840 
2841   // get current pc information
2842   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2843   int pc_for_athrow_offset = __ offset();
2844   InternalAddress pc_for_athrow(__ pc());
2845   __ lea(exceptionPC->as_register(), pc_for_athrow);
2846   add_call_info(pc_for_athrow_offset, info); // for exception handler
2847 
2848   __ verify_not_null_oop(rax);
2849   // search an exception handler (rax: exception oop, rdx: throwing pc)
2850   if (compilation()->has_fpu_code()) {
2851     unwind_id = Runtime1::handle_exception_id;
2852   } else {
2853     unwind_id = Runtime1::handle_exception_nofpu_id;
2854   }
2855   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2856 
2857   // enough room for two byte trap
2858   __ nop();
2859 }
2860 
2861 
2862 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2863   assert(exceptionOop->as_register() == rax, "must match");
2864 
2865   __ jmp(_unwind_handler_entry);
2866 }
2867 
2868 
2869 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2870 
2871   // optimized version for linear scan:
2872   // * count must be already in ECX (guaranteed by LinearScan)
2873   // * left and dest must be equal
2874   // * tmp must be unused
2875   assert(count->as_register() == SHIFT_count, "count must be in ECX");
2876   assert(left == dest, "left and dest must be equal");
2877   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2878 
2879   if (left->is_single_cpu()) {
2880     Register value = left->as_register();
2881     assert(value != SHIFT_count, "left cannot be ECX");
2882 
2883     switch (code) {
2884       case lir_shl:  __ shll(value); break;
2885       case lir_shr:  __ sarl(value); break;
2886       case lir_ushr: __ shrl(value); break;
2887       default: ShouldNotReachHere();
2888     }
2889   } else if (left->is_double_cpu()) {
2890     Register lo = left->as_register_lo();
2891     Register hi = left->as_register_hi();
2892     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2893 #ifdef _LP64
2894     switch (code) {
2895       case lir_shl:  __ shlptr(lo);        break;
2896       case lir_shr:  __ sarptr(lo);        break;
2897       case lir_ushr: __ shrptr(lo);        break;
2898       default: ShouldNotReachHere();
2899     }
2900 #else
2901 
2902     switch (code) {
2903       case lir_shl:  __ lshl(hi, lo);        break;
2904       case lir_shr:  __ lshr(hi, lo, true);  break;
2905       case lir_ushr: __ lshr(hi, lo, false); break;
2906       default: ShouldNotReachHere();
2907     }
2908 #endif // LP64
2909   } else {
2910     ShouldNotReachHere();
2911   }
2912 }
2913 
2914 
2915 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2916   if (dest->is_single_cpu()) {
2917     // first move left into dest so that left is not destroyed by the shift
2918     Register value = dest->as_register();
2919     count = count & 0x1F; // Java spec
2920 
2921     move_regs(left->as_register(), value);
2922     switch (code) {
2923       case lir_shl:  __ shll(value, count); break;
2924       case lir_shr:  __ sarl(value, count); break;
2925       case lir_ushr: __ shrl(value, count); break;
2926       default: ShouldNotReachHere();
2927     }
2928   } else if (dest->is_double_cpu()) {
2929 #ifndef _LP64
2930     Unimplemented();
2931 #else
2932     // first move left into dest so that left is not destroyed by the shift
2933     Register value = dest->as_register_lo();
2934     count = count & 0x1F; // Java spec
2935 
2936     move_regs(left->as_register_lo(), value);
2937     switch (code) {
2938       case lir_shl:  __ shlptr(value, count); break;
2939       case lir_shr:  __ sarptr(value, count); break;
2940       case lir_ushr: __ shrptr(value, count); break;
2941       default: ShouldNotReachHere();
2942     }
2943 #endif // _LP64
2944   } else {
2945     ShouldNotReachHere();
2946   }
2947 }
2948 
2949 
2950 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2951   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2952   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2953   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2954   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2955 }
2956 
2957 
2958 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2959   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2960   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2961   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2962   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2963 }
2964 
2965 
2966 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2967   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2968   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2969   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2970   __ movoop (Address(rsp, offset_from_rsp_in_bytes), o);
2971 }
2972 
2973 
2974 // This code replaces a call to arraycopy; no exception may
2975 // be thrown in this code, they must be thrown in the System.arraycopy
2976 // activation frame; we could save some checks if this would not be the case
2977 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2978   ciArrayKlass* default_type = op->expected_type();
2979   Register src = op->src()->as_register();
2980   Register dst = op->dst()->as_register();
2981   Register src_pos = op->src_pos()->as_register();
2982   Register dst_pos = op->dst_pos()->as_register();
2983   Register length  = op->length()->as_register();
2984   Register tmp = op->tmp()->as_register();
2985 
2986   CodeStub* stub = op->stub();
2987   int flags = op->flags();
2988   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2989   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2990 
2991   // if we don't know anything, just go through the generic arraycopy
2992   if (default_type == NULL) {
2993     Label done;
2994     // save outgoing arguments on stack in case call to System.arraycopy is needed
2995     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2996     // for interpreter calling conventions. Now we have to do it in new style conventions.
2997     // For the moment until C1 gets the new register allocator I just force all the
2998     // args to the right place (except the register args) and then on the back side
2999     // reload the register args properly if we go slow path. Yuck
3000 
3001     // These are proper for the calling convention
3002     store_parameter(length, 2);
3003     store_parameter(dst_pos, 1);
3004     store_parameter(dst, 0);
3005 
3006     // these are just temporary placements until we need to reload
3007     store_parameter(src_pos, 3);
3008     store_parameter(src, 4);
3009     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3010 
3011     address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
3012 
3013     address copyfunc_addr = StubRoutines::generic_arraycopy();
3014 
3015     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
3016 #ifdef _LP64
3017     // The arguments are in java calling convention so we can trivially shift them to C
3018     // convention
3019     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
3020     __ mov(c_rarg0, j_rarg0);
3021     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
3022     __ mov(c_rarg1, j_rarg1);
3023     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
3024     __ mov(c_rarg2, j_rarg2);
3025     assert_different_registers(c_rarg3, j_rarg4);
3026     __ mov(c_rarg3, j_rarg3);
3027 #ifdef _WIN64
3028     // Allocate abi space for args but be sure to keep stack aligned
3029     __ subptr(rsp, 6*wordSize);
3030     store_parameter(j_rarg4, 4);
3031     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
3032       __ call(RuntimeAddress(C_entry));
3033     } else {
3034 #ifndef PRODUCT
3035       if (PrintC1Statistics) {
3036         __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3037       }
3038 #endif
3039       __ call(RuntimeAddress(copyfunc_addr));
3040     }
3041     __ addptr(rsp, 6*wordSize);
3042 #else
3043     __ mov(c_rarg4, j_rarg4);
3044     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
3045       __ call(RuntimeAddress(C_entry));
3046     } else {
3047 #ifndef PRODUCT
3048       if (PrintC1Statistics) {
3049         __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3050       }
3051 #endif
3052       __ call(RuntimeAddress(copyfunc_addr));
3053     }
3054 #endif // _WIN64
3055 #else
3056     __ push(length);
3057     __ push(dst_pos);
3058     __ push(dst);
3059     __ push(src_pos);
3060     __ push(src);
3061 
3062     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
3063       __ call_VM_leaf(C_entry, 5); // removes pushed parameter from the stack
3064     } else {
3065 #ifndef PRODUCT
3066       if (PrintC1Statistics) {
3067         __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3068       }
3069 #endif
3070       __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
3071     }
3072 
3073 #endif // _LP64
3074 
3075     __ cmpl(rax, 0);
3076     __ jcc(Assembler::equal, *stub->continuation());
3077 
3078     if (copyfunc_addr != NULL) {
3079       __ mov(tmp, rax);
3080       __ xorl(tmp, -1);
3081     }
3082 
3083     // Reload values from the stack so they are where the stub
3084     // expects them.
3085     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3086     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3087     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3088     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3089     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3090 
3091     if (copyfunc_addr != NULL) {
3092       __ subl(length, tmp);
3093       __ addl(src_pos, tmp);
3094       __ addl(dst_pos, tmp);
3095     }
3096     __ jmp(*stub->entry());
3097 
3098     __ bind(*stub->continuation());
3099     return;
3100   }
3101 
3102   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3103 
3104   int elem_size = type2aelembytes(basic_type);
3105   Address::ScaleFactor scale;
3106 
3107   switch (elem_size) {
3108     case 1 :
3109       scale = Address::times_1;
3110       break;
3111     case 2 :
3112       scale = Address::times_2;
3113       break;
3114     case 4 :
3115       scale = Address::times_4;
3116       break;
3117     case 8 :
3118       scale = Address::times_8;
3119       break;
3120     default:
3121       scale = Address::no_scale;
3122       ShouldNotReachHere();
3123   }
3124 
3125   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3126   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3127   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3128   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3129 
3130   // length and pos's are all sign extended at this point on 64bit
3131 
3132   // test for NULL
3133   if (flags & LIR_OpArrayCopy::src_null_check) {
3134     __ testptr(src, src);
3135     __ jcc(Assembler::zero, *stub->entry());
3136   }
3137   if (flags & LIR_OpArrayCopy::dst_null_check) {
3138     __ testptr(dst, dst);
3139     __ jcc(Assembler::zero, *stub->entry());
3140   }
3141 
3142   // check if negative
3143   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3144     __ testl(src_pos, src_pos);
3145     __ jcc(Assembler::less, *stub->entry());
3146   }
3147   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3148     __ testl(dst_pos, dst_pos);
3149     __ jcc(Assembler::less, *stub->entry());
3150   }
3151 
3152   if (flags & LIR_OpArrayCopy::src_range_check) {
3153     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3154     __ cmpl(tmp, src_length_addr);
3155     __ jcc(Assembler::above, *stub->entry());
3156   }
3157   if (flags & LIR_OpArrayCopy::dst_range_check) {
3158     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3159     __ cmpl(tmp, dst_length_addr);
3160     __ jcc(Assembler::above, *stub->entry());
3161   }
3162 
3163   if (flags & LIR_OpArrayCopy::length_positive_check) {
3164     __ testl(length, length);
3165     __ jcc(Assembler::less, *stub->entry());
3166     __ jcc(Assembler::zero, *stub->continuation());
3167   }
3168 
3169 #ifdef _LP64
3170   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3171   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3172 #endif
3173 
3174   if (flags & LIR_OpArrayCopy::type_check) {
3175     // We don't know the array types are compatible
3176     if (basic_type != T_OBJECT) {
3177       // Simple test for basic type arrays
3178       if (UseCompressedClassPointers) {
3179         __ movl(tmp, src_klass_addr);
3180         __ cmpl(tmp, dst_klass_addr);
3181       } else {
3182         __ movptr(tmp, src_klass_addr);
3183         __ cmpptr(tmp, dst_klass_addr);
3184       }
3185       __ jcc(Assembler::notEqual, *stub->entry());
3186     } else {
3187       // For object arrays, if src is a sub class of dst then we can
3188       // safely do the copy.
3189       Label cont, slow;
3190 
3191       __ push(src);
3192       __ push(dst);
3193 
3194       __ load_klass(src, src);
3195       __ load_klass(dst, dst);
3196 
3197       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
3198 
3199       __ push(src);
3200       __ push(dst);
3201       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3202       __ pop(dst);
3203       __ pop(src);
3204 
3205       __ cmpl(src, 0);
3206       __ jcc(Assembler::notEqual, cont);
3207 
3208       __ bind(slow);
3209       __ pop(dst);
3210       __ pop(src);
3211 
3212       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
3213       if (copyfunc_addr != NULL) { // use stub if available
3214         // src is not a sub class of dst so we have to do a
3215         // per-element check.
3216 
3217         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
3218         if ((flags & mask) != mask) {
3219           // Check that at least both of them object arrays.
3220           assert(flags & mask, "one of the two should be known to be an object array");
3221 
3222           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3223             __ load_klass(tmp, src);
3224           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3225             __ load_klass(tmp, dst);
3226           }
3227           int lh_offset = in_bytes(Klass::layout_helper_offset());
3228           Address klass_lh_addr(tmp, lh_offset);
3229           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3230           __ cmpl(klass_lh_addr, objArray_lh);
3231           __ jcc(Assembler::notEqual, *stub->entry());
3232         }
3233 
3234        // Spill because stubs can use any register they like and it's
3235        // easier to restore just those that we care about.
3236        store_parameter(dst, 0);
3237        store_parameter(dst_pos, 1);
3238        store_parameter(length, 2);
3239        store_parameter(src_pos, 3);
3240        store_parameter(src, 4);
3241 
3242 #ifndef _LP64
3243         __ movptr(tmp, dst_klass_addr);
3244         __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
3245         __ push(tmp);
3246         __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3247         __ push(tmp);
3248         __ push(length);
3249         __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3250         __ push(tmp);
3251         __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3252         __ push(tmp);
3253 
3254         __ call_VM_leaf(copyfunc_addr, 5);
3255 #else
3256         __ movl2ptr(length, length); //higher 32bits must be null
3257 
3258         __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3259         assert_different_registers(c_rarg0, dst, dst_pos, length);
3260         __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3261         assert_different_registers(c_rarg1, dst, length);
3262 
3263         __ mov(c_rarg2, length);
3264         assert_different_registers(c_rarg2, dst);
3265 
3266 #ifdef _WIN64
3267         // Allocate abi space for args but be sure to keep stack aligned
3268         __ subptr(rsp, 6*wordSize);
3269         __ load_klass(c_rarg3, dst);
3270         __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
3271         store_parameter(c_rarg3, 4);
3272         __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
3273         __ call(RuntimeAddress(copyfunc_addr));
3274         __ addptr(rsp, 6*wordSize);
3275 #else
3276         __ load_klass(c_rarg4, dst);
3277         __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
3278         __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
3279         __ call(RuntimeAddress(copyfunc_addr));
3280 #endif
3281 
3282 #endif
3283 
3284 #ifndef PRODUCT
3285         if (PrintC1Statistics) {
3286           Label failed;
3287           __ testl(rax, rax);
3288           __ jcc(Assembler::notZero, failed);
3289           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
3290           __ bind(failed);
3291         }
3292 #endif
3293 
3294         __ testl(rax, rax);
3295         __ jcc(Assembler::zero, *stub->continuation());
3296 
3297 #ifndef PRODUCT
3298         if (PrintC1Statistics) {
3299           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
3300         }
3301 #endif
3302 
3303         __ mov(tmp, rax);
3304 
3305         __ xorl(tmp, -1);
3306 
3307         // Restore previously spilled arguments
3308         __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3309         __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3310         __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3311         __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3312         __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3313 
3314 
3315         __ subl(length, tmp);
3316         __ addl(src_pos, tmp);
3317         __ addl(dst_pos, tmp);
3318       }
3319 
3320       __ jmp(*stub->entry());
3321 
3322       __ bind(cont);
3323       __ pop(dst);
3324       __ pop(src);
3325     }
3326   }
3327 
3328 #ifdef ASSERT
3329   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3330     // Sanity check the known type with the incoming class.  For the
3331     // primitive case the types must match exactly with src.klass and
3332     // dst.klass each exactly matching the default type.  For the
3333     // object array case, if no type check is needed then either the
3334     // dst type is exactly the expected type and the src type is a
3335     // subtype which we can't check or src is the same array as dst
3336     // but not necessarily exactly of type default_type.
3337     Label known_ok, halt;
3338     __ mov_metadata(tmp, default_type->constant_encoding());
3339 #ifdef _LP64
3340     if (UseCompressedClassPointers) {
3341       __ encode_klass_not_null(tmp);
3342     }
3343 #endif
3344 
3345     if (basic_type != T_OBJECT) {
3346 
3347       if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3348       else                   __ cmpptr(tmp, dst_klass_addr);
3349       __ jcc(Assembler::notEqual, halt);
3350       if (UseCompressedClassPointers)          __ cmpl(tmp, src_klass_addr);
3351       else                   __ cmpptr(tmp, src_klass_addr);
3352       __ jcc(Assembler::equal, known_ok);
3353     } else {
3354       if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3355       else                   __ cmpptr(tmp, dst_klass_addr);
3356       __ jcc(Assembler::equal, known_ok);
3357       __ cmpptr(src, dst);
3358       __ jcc(Assembler::equal, known_ok);
3359     }
3360     __ bind(halt);
3361     __ stop("incorrect type information in arraycopy");
3362     __ bind(known_ok);
3363   }
3364 #endif
3365 
3366 #ifndef PRODUCT
3367   if (PrintC1Statistics) {
3368     __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
3369   }
3370 #endif
3371 
3372 #ifdef _LP64
3373   assert_different_registers(c_rarg0, dst, dst_pos, length);
3374   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3375   assert_different_registers(c_rarg1, length);
3376   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3377   __ mov(c_rarg2, length);
3378 
3379 #else
3380   __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3381   store_parameter(tmp, 0);
3382   __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3383   store_parameter(tmp, 1);
3384   store_parameter(length, 2);
3385 #endif // _LP64
3386 
3387   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3388   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3389   const char *name;
3390   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3391   __ call_VM_leaf(entry, 0);
3392 
3393   __ bind(*stub->continuation());
3394 }
3395 
3396 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3397   assert(op->crc()->is_single_cpu(),  "crc must be register");
3398   assert(op->val()->is_single_cpu(),  "byte value must be register");
3399   assert(op->result_opr()->is_single_cpu(), "result must be register");
3400   Register crc = op->crc()->as_register();
3401   Register val = op->val()->as_register();
3402   Register res = op->result_opr()->as_register();
3403 
3404   assert_different_registers(val, crc, res);
3405 
3406   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
3407   __ notl(crc); // ~crc
3408   __ update_byte_crc32(crc, val, res);
3409   __ notl(crc); // ~crc
3410   __ mov(res, crc);
3411 }
3412 
3413 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3414   Register obj = op->obj_opr()->as_register();  // may not be an oop
3415   Register hdr = op->hdr_opr()->as_register();
3416   Register lock = op->lock_opr()->as_register();
3417   if (!UseFastLocking) {
3418     __ jmp(*op->stub()->entry());
3419   } else if (op->code() == lir_lock) {
3420     Register scratch = noreg;
3421     if (UseBiasedLocking) {
3422       scratch = op->scratch_opr()->as_register();
3423     }
3424     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3425     // add debug info for NullPointerException only if one is possible
3426     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
3427     if (op->info() != NULL) {
3428       add_debug_info_for_null_check(null_check_offset, op->info());
3429     }
3430     // done
3431   } else if (op->code() == lir_unlock) {
3432     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3433     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3434   } else {
3435     Unimplemented();
3436   }
3437   __ bind(*op->stub()->continuation());
3438 }
3439 
3440 
3441 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3442   ciMethod* method = op->profiled_method();
3443   int bci          = op->profiled_bci();
3444   ciMethod* callee = op->profiled_callee();
3445 
3446   // Update counter for all call types
3447   ciMethodData* md = method->method_data_or_null();
3448   assert(md != NULL, "Sanity");
3449   ciProfileData* data = md->bci_to_data(bci);
3450   assert(data->is_CounterData(), "need CounterData for calls");
3451   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3452   Register mdo  = op->mdo()->as_register();
3453   __ mov_metadata(mdo, md->constant_encoding());
3454   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3455   Bytecodes::Code bc = method->java_code_at_bci(bci);
3456   const bool callee_is_static = callee->is_loaded() && callee->is_static();
3457   // Perform additional virtual call profiling for invokevirtual and
3458   // invokeinterface bytecodes
3459   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
3460       !callee_is_static &&  // required for optimized MH invokes
3461       C1ProfileVirtualCalls) {
3462     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3463     Register recv = op->recv()->as_register();
3464     assert_different_registers(mdo, recv);
3465     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3466     ciKlass* known_klass = op->known_holder();
3467     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3468       // We know the type that will be seen at this call site; we can
3469       // statically update the MethodData* rather than needing to do
3470       // dynamic tests on the receiver type
3471 
3472       // NOTE: we should probably put a lock around this search to
3473       // avoid collisions by concurrent compilations
3474       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3475       uint i;
3476       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3477         ciKlass* receiver = vc_data->receiver(i);
3478         if (known_klass->equals(receiver)) {
3479           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3480           __ addptr(data_addr, DataLayout::counter_increment);
3481           return;
3482         }
3483       }
3484 
3485       // Receiver type not found in profile data; select an empty slot
3486 
3487       // Note that this is less efficient than it should be because it
3488       // always does a write to the receiver part of the
3489       // VirtualCallData rather than just the first time
3490       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3491         ciKlass* receiver = vc_data->receiver(i);
3492         if (receiver == NULL) {
3493           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3494           __ mov_metadata(recv_addr, known_klass->constant_encoding());
3495           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3496           __ addptr(data_addr, DataLayout::counter_increment);
3497           return;
3498         }
3499       }
3500     } else {
3501       __ load_klass(recv, recv);
3502       Label update_done;
3503       type_profile_helper(mdo, md, data, recv, &update_done);
3504       // Receiver did not match any saved receiver and there is no empty row for it.
3505       // Increment total counter to indicate polymorphic case.
3506       __ addptr(counter_addr, DataLayout::counter_increment);
3507 
3508       __ bind(update_done);
3509     }
3510   } else {
3511     // Static call
3512     __ addptr(counter_addr, DataLayout::counter_increment);
3513   }
3514 }
3515 
3516 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3517   Register obj = op->obj()->as_register();
3518   Register tmp = op->tmp()->as_pointer_register();
3519   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3520   ciKlass* exact_klass = op->exact_klass();
3521   intptr_t current_klass = op->current_klass();
3522   bool not_null = op->not_null();
3523   bool no_conflict = op->no_conflict();
3524 
3525   Label update, next, none;
3526 
3527   bool do_null = !not_null;
3528   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3529   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3530 
3531   assert(do_null || do_update, "why are we here?");
3532   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3533 
3534   __ verify_oop(obj);
3535 
3536   if (tmp != obj) {
3537     __ mov(tmp, obj);
3538   }
3539   if (do_null) {
3540     __ testptr(tmp, tmp);
3541     __ jccb(Assembler::notZero, update);
3542     if (!TypeEntries::was_null_seen(current_klass)) {
3543       __ orptr(mdo_addr, TypeEntries::null_seen);
3544     }
3545     if (do_update) {
3546 #ifndef ASSERT
3547       __ jmpb(next);
3548     }
3549 #else
3550       __ jmp(next);
3551     }
3552   } else {
3553     __ testptr(tmp, tmp);
3554     __ jccb(Assembler::notZero, update);
3555     __ stop("unexpect null obj");
3556 #endif
3557   }
3558 
3559   __ bind(update);
3560 
3561   if (do_update) {
3562 #ifdef ASSERT
3563     if (exact_klass != NULL) {
3564       Label ok;
3565       __ load_klass(tmp, tmp);
3566       __ push(tmp);
3567       __ mov_metadata(tmp, exact_klass->constant_encoding());
3568       __ cmpptr(tmp, Address(rsp, 0));
3569       __ jccb(Assembler::equal, ok);
3570       __ stop("exact klass and actual klass differ");
3571       __ bind(ok);
3572       __ pop(tmp);
3573     }
3574 #endif
3575     if (!no_conflict) {
3576       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
3577         if (exact_klass != NULL) {
3578           __ mov_metadata(tmp, exact_klass->constant_encoding());
3579         } else {
3580           __ load_klass(tmp, tmp);
3581         }
3582 
3583         __ xorptr(tmp, mdo_addr);
3584         __ testptr(tmp, TypeEntries::type_klass_mask);
3585         // klass seen before, nothing to do. The unknown bit may have been
3586         // set already but no need to check.
3587         __ jccb(Assembler::zero, next);
3588 
3589         __ testptr(tmp, TypeEntries::type_unknown);
3590         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3591 
3592         if (TypeEntries::is_type_none(current_klass)) {
3593           __ cmpptr(mdo_addr, 0);
3594           __ jccb(Assembler::equal, none);
3595           __ cmpptr(mdo_addr, TypeEntries::null_seen);
3596           __ jccb(Assembler::equal, none);
3597           // There is a chance that the checks above (re-reading profiling
3598           // data from memory) fail if another thread has just set the
3599           // profiling to this obj's klass
3600           __ xorptr(tmp, mdo_addr);
3601           __ testptr(tmp, TypeEntries::type_klass_mask);
3602           __ jccb(Assembler::zero, next);
3603         }
3604       } else {
3605         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3606                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3607 
3608         __ movptr(tmp, mdo_addr);
3609         __ testptr(tmp, TypeEntries::type_unknown);
3610         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3611       }
3612 
3613       // different than before. Cannot keep accurate profile.
3614       __ orptr(mdo_addr, TypeEntries::type_unknown);
3615 
3616       if (TypeEntries::is_type_none(current_klass)) {
3617         __ jmpb(next);
3618 
3619         __ bind(none);
3620         // first time here. Set profile type.
3621         __ movptr(mdo_addr, tmp);
3622       }
3623     } else {
3624       // There's a single possible klass at this profile point
3625       assert(exact_klass != NULL, "should be");
3626       if (TypeEntries::is_type_none(current_klass)) {
3627         __ mov_metadata(tmp, exact_klass->constant_encoding());
3628         __ xorptr(tmp, mdo_addr);
3629         __ testptr(tmp, TypeEntries::type_klass_mask);
3630 #ifdef ASSERT
3631         __ jcc(Assembler::zero, next);
3632 
3633         {
3634           Label ok;
3635           __ push(tmp);
3636           __ cmpptr(mdo_addr, 0);
3637           __ jcc(Assembler::equal, ok);
3638           __ cmpptr(mdo_addr, TypeEntries::null_seen);
3639           __ jcc(Assembler::equal, ok);
3640           // may have been set by another thread
3641           __ mov_metadata(tmp, exact_klass->constant_encoding());
3642           __ xorptr(tmp, mdo_addr);
3643           __ testptr(tmp, TypeEntries::type_mask);
3644           __ jcc(Assembler::zero, ok);
3645 
3646           __ stop("unexpected profiling mismatch");
3647           __ bind(ok);
3648           __ pop(tmp);
3649         }
3650 #else
3651         __ jccb(Assembler::zero, next);
3652 #endif
3653         // first time here. Set profile type.
3654         __ movptr(mdo_addr, tmp);
3655       } else {
3656         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3657                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3658 
3659         __ movptr(tmp, mdo_addr);
3660         __ testptr(tmp, TypeEntries::type_unknown);
3661         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3662 
3663         __ orptr(mdo_addr, TypeEntries::type_unknown);
3664       }
3665     }
3666 
3667     __ bind(next);
3668   }
3669 }
3670 
3671 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3672   Unimplemented();
3673 }
3674 
3675 
3676 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3677   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3678 }
3679 
3680 
3681 void LIR_Assembler::align_backward_branch_target() {
3682   __ align(BytesPerWord);
3683 }
3684 
3685 
3686 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
3687   if (left->is_single_cpu()) {
3688     __ negl(left->as_register());
3689     move_regs(left->as_register(), dest->as_register());
3690 
3691   } else if (left->is_double_cpu()) {
3692     Register lo = left->as_register_lo();
3693 #ifdef _LP64
3694     Register dst = dest->as_register_lo();
3695     __ movptr(dst, lo);
3696     __ negptr(dst);
3697 #else
3698     Register hi = left->as_register_hi();
3699     __ lneg(hi, lo);
3700     if (dest->as_register_lo() == hi) {
3701       assert(dest->as_register_hi() != lo, "destroying register");
3702       move_regs(hi, dest->as_register_hi());
3703       move_regs(lo, dest->as_register_lo());
3704     } else {
3705       move_regs(lo, dest->as_register_lo());
3706       move_regs(hi, dest->as_register_hi());
3707     }
3708 #endif // _LP64
3709 
3710   } else if (dest->is_single_xmm()) {
3711     if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3712       __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3713     }
3714     if (UseAVX > 0) {
3715       __ vnegatess(dest->as_xmm_float_reg(), dest->as_xmm_float_reg(),
3716                    ExternalAddress((address)float_signflip_pool));
3717     } else {
3718       __ xorps(dest->as_xmm_float_reg(),
3719                ExternalAddress((address)float_signflip_pool));
3720     }
3721   } else if (dest->is_double_xmm()) {
3722     if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3723       __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3724     }
3725     if (UseAVX > 0) {
3726       __ vnegatesd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg(),
3727                    ExternalAddress((address)double_signflip_pool));
3728     } else {
3729       __ xorpd(dest->as_xmm_double_reg(),
3730                ExternalAddress((address)double_signflip_pool));
3731     }
3732   } else if (left->is_single_fpu() || left->is_double_fpu()) {
3733     assert(left->fpu() == 0, "arg must be on TOS");
3734     assert(dest->fpu() == 0, "dest must be TOS");
3735     __ fchs();
3736 
3737   } else {
3738     ShouldNotReachHere();
3739   }
3740 }
3741 
3742 
3743 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
3744   assert(addr->is_address() && dest->is_register(), "check");
3745   Register reg;
3746   reg = dest->as_pointer_register();
3747   __ lea(reg, as_Address(addr->as_address_ptr()));
3748 }
3749 
3750 
3751 
3752 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3753   assert(!tmp->is_valid(), "don't need temporary");
3754   __ call(RuntimeAddress(dest));
3755   if (info != NULL) {
3756     add_call_info_here(info);
3757   }
3758 }
3759 
3760 
3761 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3762   assert(type == T_LONG, "only for volatile long fields");
3763 
3764   if (info != NULL) {
3765     add_debug_info_for_null_check_here(info);
3766   }
3767 
3768   if (src->is_double_xmm()) {
3769     if (dest->is_double_cpu()) {
3770 #ifdef _LP64
3771       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3772 #else
3773       __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
3774       __ psrlq(src->as_xmm_double_reg(), 32);
3775       __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
3776 #endif // _LP64
3777     } else if (dest->is_double_stack()) {
3778       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3779     } else if (dest->is_address()) {
3780       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3781     } else {
3782       ShouldNotReachHere();
3783     }
3784 
3785   } else if (dest->is_double_xmm()) {
3786     if (src->is_double_stack()) {
3787       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3788     } else if (src->is_address()) {
3789       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3790     } else {
3791       ShouldNotReachHere();
3792     }
3793 
3794   } else if (src->is_double_fpu()) {
3795     assert(src->fpu_regnrLo() == 0, "must be TOS");
3796     if (dest->is_double_stack()) {
3797       __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
3798     } else if (dest->is_address()) {
3799       __ fistp_d(as_Address(dest->as_address_ptr()));
3800     } else {
3801       ShouldNotReachHere();
3802     }
3803 
3804   } else if (dest->is_double_fpu()) {
3805     assert(dest->fpu_regnrLo() == 0, "must be TOS");
3806     if (src->is_double_stack()) {
3807       __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
3808     } else if (src->is_address()) {
3809       __ fild_d(as_Address(src->as_address_ptr()));
3810     } else {
3811       ShouldNotReachHere();
3812     }
3813   } else {
3814     ShouldNotReachHere();
3815   }
3816 }
3817 
3818 #ifdef ASSERT
3819 // emit run-time assertion
3820 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3821   assert(op->code() == lir_assert, "must be");
3822 
3823   if (op->in_opr1()->is_valid()) {
3824     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3825     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3826   } else {
3827     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3828     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3829   }
3830 
3831   Label ok;
3832   if (op->condition() != lir_cond_always) {
3833     Assembler::Condition acond = Assembler::zero;
3834     switch (op->condition()) {
3835       case lir_cond_equal:        acond = Assembler::equal;       break;
3836       case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
3837       case lir_cond_less:         acond = Assembler::less;        break;
3838       case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
3839       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3840       case lir_cond_greater:      acond = Assembler::greater;     break;
3841       case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
3842       case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
3843       default:                    ShouldNotReachHere();
3844     }
3845     __ jcc(acond, ok);
3846   }
3847   if (op->halt()) {
3848     const char* str = __ code_string(op->msg());
3849     __ stop(str);
3850   } else {
3851     breakpoint();
3852   }
3853   __ bind(ok);
3854 }
3855 #endif
3856 
3857 void LIR_Assembler::membar() {
3858   // QQQ sparc TSO uses this,
3859   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3860 }
3861 
3862 void LIR_Assembler::membar_acquire() {
3863   // No x86 machines currently require load fences
3864 }
3865 
3866 void LIR_Assembler::membar_release() {
3867   // No x86 machines currently require store fences
3868 }
3869 
3870 void LIR_Assembler::membar_loadload() {
3871   // no-op
3872   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3873 }
3874 
3875 void LIR_Assembler::membar_storestore() {
3876   // no-op
3877   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3878 }
3879 
3880 void LIR_Assembler::membar_loadstore() {
3881   // no-op
3882   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3883 }
3884 
3885 void LIR_Assembler::membar_storeload() {
3886   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3887 }
3888 
3889 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3890   assert(result_reg->is_register(), "check");
3891 #ifdef _LP64
3892   // __ get_thread(result_reg->as_register_lo());
3893   __ mov(result_reg->as_register(), r15_thread);
3894 #else
3895   __ get_thread(result_reg->as_register());
3896 #endif // _LP64
3897 }
3898 
3899 
3900 void LIR_Assembler::peephole(LIR_List*) {
3901   // do nothing for now
3902 }
3903 
3904 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3905   assert(data == dest, "xchg/xadd uses only 2 operands");
3906 
3907   if (data->type() == T_INT) {
3908     if (code == lir_xadd) {
3909       if (os::is_MP()) {
3910         __ lock();
3911       }
3912       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3913     } else {
3914       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3915     }
3916   } else if (data->is_oop()) {
3917     assert (code == lir_xchg, "xadd for oops");
3918     Register obj = data->as_register();
3919 #ifdef _LP64
3920     if (UseCompressedOops) {
3921       __ encode_heap_oop(obj);
3922       __ xchgl(obj, as_Address(src->as_address_ptr()));
3923       __ decode_heap_oop(obj);
3924     } else {
3925       __ xchgptr(obj, as_Address(src->as_address_ptr()));
3926     }
3927 #else
3928     __ xchgl(obj, as_Address(src->as_address_ptr()));
3929 #endif
3930   } else if (data->type() == T_LONG) {
3931 #ifdef _LP64
3932     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3933     if (code == lir_xadd) {
3934       if (os::is_MP()) {
3935         __ lock();
3936       }
3937       __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3938     } else {
3939       __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3940     }
3941 #else
3942     ShouldNotReachHere();
3943 #endif
3944   } else {
3945     ShouldNotReachHere();
3946   }
3947 }
3948 
3949 #undef __