1 /*
   2  * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "c1/c1_Compilation.hpp"
  28 #include "c1/c1_LIRAssembler.hpp"
  29 #include "c1/c1_MacroAssembler.hpp"
  30 #include "c1/c1_Runtime1.hpp"
  31 #include "c1/c1_ValueStack.hpp"
  32 #include "ci/ciArrayKlass.hpp"
  33 #include "ci/ciInstance.hpp"
  34 #include "gc_interface/collectedHeap.hpp"
  35 #include "memory/barrierSet.hpp"
  36 #include "memory/cardTableModRefBS.hpp"
  37 #include "nativeInst_x86.hpp"
  38 #include "oops/objArrayKlass.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 
  41 
  42 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  43 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  44 // fast versions of NegF/NegD and AbsF/AbsD.
  45 
  46 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  47 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  48   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  49   // of 128-bits operands for SSE instructions.
  50   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  51   // Store the value to a 128-bits operand.
  52   operand[0] = lo;
  53   operand[1] = hi;
  54   return operand;
  55 }
  56 
  57 // Buffer for 128-bits masks used by SSE instructions.
  58 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  59 
  60 // Static initialization during VM startup.
  61 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
  62 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
  63 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));
  64 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
  65 
  66 
  67 
  68 NEEDS_CLEANUP // remove this definitions ?
  69 const Register IC_Klass    = rax;   // where the IC klass is cached
  70 const Register SYNC_header = rax;   // synchronization header
  71 const Register SHIFT_count = rcx;   // where count for shift operations must be
  72 
  73 #define __ _masm->
  74 
  75 
  76 static void select_different_registers(Register preserve,
  77                                        Register extra,
  78                                        Register &tmp1,
  79                                        Register &tmp2) {
  80   if (tmp1 == preserve) {
  81     assert_different_registers(tmp1, tmp2, extra);
  82     tmp1 = extra;
  83   } else if (tmp2 == preserve) {
  84     assert_different_registers(tmp1, tmp2, extra);
  85     tmp2 = extra;
  86   }
  87   assert_different_registers(preserve, tmp1, tmp2);
  88 }
  89 
  90 
  91 
  92 static void select_different_registers(Register preserve,
  93                                        Register extra,
  94                                        Register &tmp1,
  95                                        Register &tmp2,
  96                                        Register &tmp3) {
  97   if (tmp1 == preserve) {
  98     assert_different_registers(tmp1, tmp2, tmp3, extra);
  99     tmp1 = extra;
 100   } else if (tmp2 == preserve) {
 101     assert_different_registers(tmp1, tmp2, tmp3, extra);
 102     tmp2 = extra;
 103   } else if (tmp3 == preserve) {
 104     assert_different_registers(tmp1, tmp2, tmp3, extra);
 105     tmp3 = extra;
 106   }
 107   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 108 }
 109 
 110 
 111 
 112 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 113   if (opr->is_constant()) {
 114     LIR_Const* constant = opr->as_constant_ptr();
 115     switch (constant->type()) {
 116       case T_INT: {
 117         return true;
 118       }
 119 
 120       default:
 121         return false;
 122     }
 123   }
 124   return false;
 125 }
 126 
 127 
 128 LIR_Opr LIR_Assembler::receiverOpr() {
 129   return FrameMap::receiver_opr;
 130 }
 131 
 132 LIR_Opr LIR_Assembler::osrBufferPointer() {
 133   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 134 }
 135 
 136 //--------------fpu register translations-----------------------
 137 
 138 
 139 address LIR_Assembler::float_constant(float f) {
 140   address const_addr = __ float_constant(f);
 141   if (const_addr == NULL) {
 142     bailout("const section overflow");
 143     return __ code()->consts()->start();
 144   } else {
 145     return const_addr;
 146   }
 147 }
 148 
 149 
 150 address LIR_Assembler::double_constant(double d) {
 151   address const_addr = __ double_constant(d);
 152   if (const_addr == NULL) {
 153     bailout("const section overflow");
 154     return __ code()->consts()->start();
 155   } else {
 156     return const_addr;
 157   }
 158 }
 159 
 160 
 161 void LIR_Assembler::set_24bit_FPU() {
 162   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
 163 }
 164 
 165 void LIR_Assembler::reset_FPU() {
 166   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 167 }
 168 
 169 void LIR_Assembler::fpop() {
 170   __ fpop();
 171 }
 172 
 173 void LIR_Assembler::fxch(int i) {
 174   __ fxch(i);
 175 }
 176 
 177 void LIR_Assembler::fld(int i) {
 178   __ fld_s(i);
 179 }
 180 
 181 void LIR_Assembler::ffree(int i) {
 182   __ ffree(i);
 183 }
 184 
 185 void LIR_Assembler::breakpoint() {
 186   __ int3();
 187 }
 188 
 189 void LIR_Assembler::push(LIR_Opr opr) {
 190   if (opr->is_single_cpu()) {
 191     __ push_reg(opr->as_register());
 192   } else if (opr->is_double_cpu()) {
 193     NOT_LP64(__ push_reg(opr->as_register_hi()));
 194     __ push_reg(opr->as_register_lo());
 195   } else if (opr->is_stack()) {
 196     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 197   } else if (opr->is_constant()) {
 198     LIR_Const* const_opr = opr->as_constant_ptr();
 199     if (const_opr->type() == T_OBJECT) {
 200       __ push_oop(const_opr->as_jobject());
 201     } else if (const_opr->type() == T_INT) {
 202       __ push_jint(const_opr->as_jint());
 203     } else {
 204       ShouldNotReachHere();
 205     }
 206 
 207   } else {
 208     ShouldNotReachHere();
 209   }
 210 }
 211 
 212 void LIR_Assembler::pop(LIR_Opr opr) {
 213   if (opr->is_single_cpu()) {
 214     __ pop_reg(opr->as_register());
 215   } else {
 216     ShouldNotReachHere();
 217   }
 218 }
 219 
 220 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 221   return addr->base()->is_illegal() && addr->index()->is_illegal();
 222 }
 223 
 224 //-------------------------------------------
 225 
 226 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 227   return as_Address(addr, rscratch1);
 228 }
 229 
 230 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 231   if (addr->base()->is_illegal()) {
 232     assert(addr->index()->is_illegal(), "must be illegal too");
 233     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 234     if (! __ reachable(laddr)) {
 235       __ movptr(tmp, laddr.addr());
 236       Address res(tmp, 0);
 237       return res;
 238     } else {
 239       return __ as_Address(laddr);
 240     }
 241   }
 242 
 243   Register base = addr->base()->as_pointer_register();
 244 
 245   if (addr->index()->is_illegal()) {
 246     return Address( base, addr->disp());
 247   } else if (addr->index()->is_cpu_register()) {
 248     Register index = addr->index()->as_pointer_register();
 249     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 250   } else if (addr->index()->is_constant()) {
 251     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 252     assert(Assembler::is_simm32(addr_offset), "must be");
 253 
 254     return Address(base, addr_offset);
 255   } else {
 256     Unimplemented();
 257     return Address();
 258   }
 259 }
 260 
 261 
 262 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 263   Address base = as_Address(addr);
 264   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 265 }
 266 
 267 
 268 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 269   return as_Address(addr);
 270 }
 271 
 272 
 273 void LIR_Assembler::osr_entry() {
 274   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 275   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 276   ValueStack* entry_state = osr_entry->state();
 277   int number_of_locks = entry_state->locks_size();
 278 
 279   // we jump here if osr happens with the interpreter
 280   // state set up to continue at the beginning of the
 281   // loop that triggered osr - in particular, we have
 282   // the following registers setup:
 283   //
 284   // rcx: osr buffer
 285   //
 286 
 287   // build frame
 288   ciMethod* m = compilation()->method();
 289   __ build_frame(initial_frame_size_in_bytes());
 290 
 291   // OSR buffer is
 292   //
 293   // locals[nlocals-1..0]
 294   // monitors[0..number_of_locks]
 295   //
 296   // locals is a direct copy of the interpreter frame so in the osr buffer
 297   // so first slot in the local array is the last local from the interpreter
 298   // and last slot is local[0] (receiver) from the interpreter
 299   //
 300   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 301   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 302   // in the interpreter frame (the method lock if a sync method)
 303 
 304   // Initialize monitors in the compiled activation.
 305   //   rcx: pointer to osr buffer
 306   //
 307   // All other registers are dead at this point and the locals will be
 308   // copied into place by code emitted in the IR.
 309 
 310   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 311   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 312     int monitor_offset = BytesPerWord * method()->max_locals() +
 313       (2 * BytesPerWord) * (number_of_locks - 1);
 314     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 315     // the OSR buffer using 2 word entries: first the lock and then
 316     // the oop.
 317     for (int i = 0; i < number_of_locks; i++) {
 318       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 319 #ifdef ASSERT
 320       // verify the interpreter's monitor has a non-null object
 321       {
 322         Label L;
 323         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD);
 324         __ jcc(Assembler::notZero, L);
 325         __ stop("locked object is NULL");
 326         __ bind(L);
 327       }
 328 #endif
 329       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 330       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 331       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 332       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 333     }
 334   }
 335 }
 336 
 337 
 338 // inline cache check; done before the frame is built.
 339 int LIR_Assembler::check_icache() {
 340   Register receiver = FrameMap::receiver_opr->as_register();
 341   Register ic_klass = IC_Klass;
 342   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
 343   const bool do_post_padding = VerifyOops || UseCompressedOops;
 344   if (!do_post_padding) {
 345     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
 346     while ((__ offset() + ic_cmp_size) % CodeEntryAlignment != 0) {
 347       __ nop();
 348     }
 349   }
 350   int offset = __ offset();
 351   __ inline_cache_check(receiver, IC_Klass);
 352   assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
 353   if (do_post_padding) {
 354     // force alignment after the cache check.
 355     // It's been verified to be aligned if !VerifyOops
 356     __ align(CodeEntryAlignment);
 357   }
 358   return offset;
 359 }
 360 
 361 
 362 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 363   jobject o = NULL;
 364   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 365   __ movoop(reg, o);
 366   patching_epilog(patch, lir_patch_normal, reg, info);
 367 }
 368 
 369 
 370 // This specifies the rsp decrement needed to build the frame
 371 int LIR_Assembler::initial_frame_size_in_bytes() {
 372   // if rounding, must let FrameMap know!
 373 
 374   // The frame_map records size in slots (32bit word)
 375 
 376   // subtract two words to account for return address and link
 377   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 378 }
 379 
 380 
 381 int LIR_Assembler::emit_exception_handler() {
 382   // if the last instruction is a call (typically to do a throw which
 383   // is coming at the end after block reordering) the return address
 384   // must still point into the code area in order to avoid assertion
 385   // failures when searching for the corresponding bci => add a nop
 386   // (was bug 5/14/1999 - gri)
 387   __ nop();
 388 
 389   // generate code for exception handler
 390   address handler_base = __ start_a_stub(exception_handler_size);
 391   if (handler_base == NULL) {
 392     // not enough space left for the handler
 393     bailout("exception handler overflow");
 394     return -1;
 395   }
 396 
 397   int offset = code_offset();
 398 
 399   // the exception oop and pc are in rax, and rdx
 400   // no other registers need to be preserved, so invalidate them
 401   __ invalidate_registers(false, true, true, false, true, true);
 402 
 403   // check that there is really an exception
 404   __ verify_not_null_oop(rax);
 405 
 406   // search an exception handler (rax: exception oop, rdx: throwing pc)
 407   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 408   __ should_not_reach_here();
 409   guarantee(code_offset() - offset <= exception_handler_size, "overflow");
 410   __ end_a_stub();
 411 
 412   return offset;
 413 }
 414 
 415 
 416 // Emit the code to remove the frame from the stack in the exception
 417 // unwind path.
 418 int LIR_Assembler::emit_unwind_handler() {
 419 #ifndef PRODUCT
 420   if (CommentedAssembly) {
 421     _masm->block_comment("Unwind handler");
 422   }
 423 #endif
 424 
 425   int offset = code_offset();
 426 
 427   // Fetch the exception from TLS and clear out exception related thread state
 428   __ get_thread(rsi);
 429   __ movptr(rax, Address(rsi, JavaThread::exception_oop_offset()));
 430   __ movptr(Address(rsi, JavaThread::exception_oop_offset()), (intptr_t)NULL_WORD);
 431   __ movptr(Address(rsi, JavaThread::exception_pc_offset()), (intptr_t)NULL_WORD);
 432 
 433   __ bind(_unwind_handler_entry);
 434   __ verify_not_null_oop(rax);
 435   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 436     __ mov(rsi, rax);  // Preserve the exception
 437   }
 438 
 439   // Preform needed unlocking
 440   MonitorExitStub* stub = NULL;
 441   if (method()->is_synchronized()) {
 442     monitor_address(0, FrameMap::rax_opr);
 443     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 444     __ unlock_object(rdi, rbx, rax, *stub->entry());
 445     __ bind(*stub->continuation());
 446   }
 447 
 448   if (compilation()->env()->dtrace_method_probes()) {
 449     __ get_thread(rax);
 450     __ movptr(Address(rsp, 0), rax);
 451     __ movoop(Address(rsp, sizeof(void*)), method()->constant_encoding());
 452     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 453   }
 454 
 455   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 456     __ mov(rax, rsi);  // Restore the exception
 457   }
 458 
 459   // remove the activation and dispatch to the unwind handler
 460   __ remove_frame(initial_frame_size_in_bytes());
 461   __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 462 
 463   // Emit the slow path assembly
 464   if (stub != NULL) {
 465     stub->emit_code(this);
 466   }
 467 
 468   return offset;
 469 }
 470 
 471 
 472 int LIR_Assembler::emit_deopt_handler() {
 473   // if the last instruction is a call (typically to do a throw which
 474   // is coming at the end after block reordering) the return address
 475   // must still point into the code area in order to avoid assertion
 476   // failures when searching for the corresponding bci => add a nop
 477   // (was bug 5/14/1999 - gri)
 478   __ nop();
 479 
 480   // generate code for exception handler
 481   address handler_base = __ start_a_stub(deopt_handler_size);
 482   if (handler_base == NULL) {
 483     // not enough space left for the handler
 484     bailout("deopt handler overflow");
 485     return -1;
 486   }
 487 
 488   int offset = code_offset();
 489   InternalAddress here(__ pc());
 490 
 491   __ pushptr(here.addr());
 492   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 493   guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
 494   __ end_a_stub();
 495 
 496   return offset;
 497 }
 498 
 499 
 500 // This is the fast version of java.lang.String.compare; it has not
 501 // OSR-entry and therefore, we generate a slow version for OSR's
 502 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
 503   __ movptr (rbx, rcx); // receiver is in rcx
 504   __ movptr (rax, arg1->as_register());
 505 
 506   // Get addresses of first characters from both Strings
 507   __ load_heap_oop(rsi, Address(rax, java_lang_String::value_offset_in_bytes()));
 508   __ movptr       (rcx, Address(rax, java_lang_String::offset_offset_in_bytes()));
 509   __ lea          (rsi, Address(rsi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 510 
 511 
 512   // rbx, may be NULL
 513   add_debug_info_for_null_check_here(info);
 514   __ load_heap_oop(rdi, Address(rbx, java_lang_String::value_offset_in_bytes()));
 515   __ movptr       (rcx, Address(rbx, java_lang_String::offset_offset_in_bytes()));
 516   __ lea          (rdi, Address(rdi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 517 
 518   // compute minimum length (in rax) and difference of lengths (on top of stack)
 519   __ movl  (rbx, Address(rbx, java_lang_String::count_offset_in_bytes()));
 520   __ movl  (rax, Address(rax, java_lang_String::count_offset_in_bytes()));
 521   __ mov   (rcx, rbx);
 522   __ subptr(rbx, rax); // subtract lengths
 523   __ push  (rbx);      // result
 524   __ cmov  (Assembler::lessEqual, rax, rcx);
 525 
 526   // is minimum length 0?
 527   Label noLoop, haveResult;
 528   __ testptr (rax, rax);
 529   __ jcc (Assembler::zero, noLoop);
 530 
 531   // compare first characters
 532   __ load_unsigned_short(rcx, Address(rdi, 0));
 533   __ load_unsigned_short(rbx, Address(rsi, 0));
 534   __ subl(rcx, rbx);
 535   __ jcc(Assembler::notZero, haveResult);
 536   // starting loop
 537   __ decrement(rax); // we already tested index: skip one
 538   __ jcc(Assembler::zero, noLoop);
 539 
 540   // set rsi.edi to the end of the arrays (arrays have same length)
 541   // negate the index
 542 
 543   __ lea(rsi, Address(rsi, rax, Address::times_2, type2aelembytes(T_CHAR)));
 544   __ lea(rdi, Address(rdi, rax, Address::times_2, type2aelembytes(T_CHAR)));
 545   __ negptr(rax);
 546 
 547   // compare the strings in a loop
 548 
 549   Label loop;
 550   __ align(wordSize);
 551   __ bind(loop);
 552   __ load_unsigned_short(rcx, Address(rdi, rax, Address::times_2, 0));
 553   __ load_unsigned_short(rbx, Address(rsi, rax, Address::times_2, 0));
 554   __ subl(rcx, rbx);
 555   __ jcc(Assembler::notZero, haveResult);
 556   __ increment(rax);
 557   __ jcc(Assembler::notZero, loop);
 558 
 559   // strings are equal up to min length
 560 
 561   __ bind(noLoop);
 562   __ pop(rax);
 563   return_op(LIR_OprFact::illegalOpr);
 564 
 565   __ bind(haveResult);
 566   // leave instruction is going to discard the TOS value
 567   __ mov (rax, rcx); // result of call is in rax,
 568 }
 569 
 570 
 571 void LIR_Assembler::return_op(LIR_Opr result) {
 572   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 573   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 574     assert(result->fpu() == 0, "result must already be on TOS");
 575   }
 576 
 577   // Pop the stack before the safepoint code
 578   __ remove_frame(initial_frame_size_in_bytes());
 579 
 580   bool result_is_oop = result->is_valid() ? result->is_oop() : false;
 581 
 582   // Note: we do not need to round double result; float result has the right precision
 583   // the poll sets the condition code, but no data registers
 584   AddressLiteral polling_page(os::get_polling_page() + (SafepointPollOffset % os::vm_page_size()),
 585                               relocInfo::poll_return_type);
 586 
 587   if (Assembler::is_polling_page_far()) {
 588     __ lea(rscratch1, polling_page);
 589     __ relocate(relocInfo::poll_return_type);
 590     __ testl(rax, Address(rscratch1, 0));
 591   } else {
 592     __ testl(rax, polling_page);
 593   }
 594   __ ret(0);
 595 }
 596 
 597 
 598 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 599   AddressLiteral polling_page(os::get_polling_page() + (SafepointPollOffset % os::vm_page_size()),
 600                               relocInfo::poll_type);
 601   guarantee(info != NULL, "Shouldn't be NULL");
 602   int offset = __ offset();
 603   if (Assembler::is_polling_page_far()) {
 604     __ lea(rscratch1, polling_page);
 605     offset = __ offset();
 606     add_debug_info_for_branch(info);
 607     __ testl(rax, Address(rscratch1, 0));
 608   } else {
 609     add_debug_info_for_branch(info);
 610     __ testl(rax, polling_page);
 611   }
 612   return offset;
 613 }
 614 
 615 
 616 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 617   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 618 }
 619 
 620 void LIR_Assembler::swap_reg(Register a, Register b) {
 621   __ xchgptr(a, b);
 622 }
 623 
 624 
 625 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 626   assert(src->is_constant(), "should not call otherwise");
 627   assert(dest->is_register(), "should not call otherwise");
 628   LIR_Const* c = src->as_constant_ptr();
 629 
 630   switch (c->type()) {
 631     case T_INT: {
 632       assert(patch_code == lir_patch_none, "no patching handled here");
 633       __ movl(dest->as_register(), c->as_jint());
 634       break;
 635     }
 636 
 637     case T_ADDRESS: {
 638       assert(patch_code == lir_patch_none, "no patching handled here");
 639       __ movptr(dest->as_register(), c->as_jint());
 640       break;
 641     }
 642 
 643     case T_LONG: {
 644       assert(patch_code == lir_patch_none, "no patching handled here");
 645 #ifdef _LP64
 646       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 647 #else
 648       __ movptr(dest->as_register_lo(), c->as_jint_lo());
 649       __ movptr(dest->as_register_hi(), c->as_jint_hi());
 650 #endif // _LP64
 651       break;
 652     }
 653 
 654     case T_OBJECT: {
 655       if (patch_code != lir_patch_none) {
 656         jobject2reg_with_patching(dest->as_register(), info);
 657       } else {
 658         __ movoop(dest->as_register(), c->as_jobject());
 659       }
 660       break;
 661     }
 662 
 663     case T_FLOAT: {
 664       if (dest->is_single_xmm()) {
 665         if (c->is_zero_float()) {
 666           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 667         } else {
 668           __ movflt(dest->as_xmm_float_reg(),
 669                    InternalAddress(float_constant(c->as_jfloat())));
 670         }
 671       } else {
 672         assert(dest->is_single_fpu(), "must be");
 673         assert(dest->fpu_regnr() == 0, "dest must be TOS");
 674         if (c->is_zero_float()) {
 675           __ fldz();
 676         } else if (c->is_one_float()) {
 677           __ fld1();
 678         } else {
 679           __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
 680         }
 681       }
 682       break;
 683     }
 684 
 685     case T_DOUBLE: {
 686       if (dest->is_double_xmm()) {
 687         if (c->is_zero_double()) {
 688           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 689         } else {
 690           __ movdbl(dest->as_xmm_double_reg(),
 691                     InternalAddress(double_constant(c->as_jdouble())));
 692         }
 693       } else {
 694         assert(dest->is_double_fpu(), "must be");
 695         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
 696         if (c->is_zero_double()) {
 697           __ fldz();
 698         } else if (c->is_one_double()) {
 699           __ fld1();
 700         } else {
 701           __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
 702         }
 703       }
 704       break;
 705     }
 706 
 707     default:
 708       ShouldNotReachHere();
 709   }
 710 }
 711 
 712 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 713   assert(src->is_constant(), "should not call otherwise");
 714   assert(dest->is_stack(), "should not call otherwise");
 715   LIR_Const* c = src->as_constant_ptr();
 716 
 717   switch (c->type()) {
 718     case T_INT:  // fall through
 719     case T_FLOAT:
 720       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 721       break;
 722 
 723     case T_ADDRESS:
 724       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 725       break;
 726 
 727     case T_OBJECT:
 728       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject());
 729       break;
 730 
 731     case T_LONG:  // fall through
 732     case T_DOUBLE:
 733 #ifdef _LP64
 734       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 735                                             lo_word_offset_in_bytes), (intptr_t)c->as_jlong_bits());
 736 #else
 737       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 738                                               lo_word_offset_in_bytes), c->as_jint_lo_bits());
 739       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 740                                               hi_word_offset_in_bytes), c->as_jint_hi_bits());
 741 #endif // _LP64
 742       break;
 743 
 744     default:
 745       ShouldNotReachHere();
 746   }
 747 }
 748 
 749 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 750   assert(src->is_constant(), "should not call otherwise");
 751   assert(dest->is_address(), "should not call otherwise");
 752   LIR_Const* c = src->as_constant_ptr();
 753   LIR_Address* addr = dest->as_address_ptr();
 754 
 755   int null_check_here = code_offset();
 756   switch (type) {
 757     case T_INT:    // fall through
 758     case T_FLOAT:
 759       __ movl(as_Address(addr), c->as_jint_bits());
 760       break;
 761 
 762     case T_ADDRESS:
 763       __ movptr(as_Address(addr), c->as_jint_bits());
 764       break;
 765 
 766     case T_OBJECT:  // fall through
 767     case T_ARRAY:
 768       if (c->as_jobject() == NULL) {
 769         if (UseCompressedOops && !wide) {
 770           __ movl(as_Address(addr), (int32_t)NULL_WORD);
 771         } else {
 772           __ movptr(as_Address(addr), NULL_WORD);
 773         }
 774       } else {
 775         if (is_literal_address(addr)) {
 776           ShouldNotReachHere();
 777           __ movoop(as_Address(addr, noreg), c->as_jobject());
 778         } else {
 779 #ifdef _LP64
 780           __ movoop(rscratch1, c->as_jobject());
 781           if (UseCompressedOops && !wide) {
 782             __ encode_heap_oop(rscratch1);
 783             null_check_here = code_offset();
 784             __ movl(as_Address_lo(addr), rscratch1);
 785           } else {
 786             null_check_here = code_offset();
 787             __ movptr(as_Address_lo(addr), rscratch1);
 788           }
 789 #else
 790           __ movoop(as_Address(addr), c->as_jobject());
 791 #endif
 792         }
 793       }
 794       break;
 795 
 796     case T_LONG:    // fall through
 797     case T_DOUBLE:
 798 #ifdef _LP64
 799       if (is_literal_address(addr)) {
 800         ShouldNotReachHere();
 801         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 802       } else {
 803         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 804         null_check_here = code_offset();
 805         __ movptr(as_Address_lo(addr), r10);
 806       }
 807 #else
 808       // Always reachable in 32bit so this doesn't produce useless move literal
 809       __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
 810       __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
 811 #endif // _LP64
 812       break;
 813 
 814     case T_BOOLEAN: // fall through
 815     case T_BYTE:
 816       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 817       break;
 818 
 819     case T_CHAR:    // fall through
 820     case T_SHORT:
 821       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 822       break;
 823 
 824     default:
 825       ShouldNotReachHere();
 826   };
 827 
 828   if (info != NULL) {
 829     add_debug_info_for_null_check(null_check_here, info);
 830   }
 831 }
 832 
 833 
 834 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 835   assert(src->is_register(), "should not call otherwise");
 836   assert(dest->is_register(), "should not call otherwise");
 837 
 838   // move between cpu-registers
 839   if (dest->is_single_cpu()) {
 840 #ifdef _LP64
 841     if (src->type() == T_LONG) {
 842       // Can do LONG -> OBJECT
 843       move_regs(src->as_register_lo(), dest->as_register());
 844       return;
 845     }
 846 #endif
 847     assert(src->is_single_cpu(), "must match");
 848     if (src->type() == T_OBJECT) {
 849       __ verify_oop(src->as_register());
 850     }
 851     move_regs(src->as_register(), dest->as_register());
 852 
 853   } else if (dest->is_double_cpu()) {
 854 #ifdef _LP64
 855     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 856       // Surprising to me but we can see move of a long to t_object
 857       __ verify_oop(src->as_register());
 858       move_regs(src->as_register(), dest->as_register_lo());
 859       return;
 860     }
 861 #endif
 862     assert(src->is_double_cpu(), "must match");
 863     Register f_lo = src->as_register_lo();
 864     Register f_hi = src->as_register_hi();
 865     Register t_lo = dest->as_register_lo();
 866     Register t_hi = dest->as_register_hi();
 867 #ifdef _LP64
 868     assert(f_hi == f_lo, "must be same");
 869     assert(t_hi == t_lo, "must be same");
 870     move_regs(f_lo, t_lo);
 871 #else
 872     assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
 873 
 874 
 875     if (f_lo == t_hi && f_hi == t_lo) {
 876       swap_reg(f_lo, f_hi);
 877     } else if (f_hi == t_lo) {
 878       assert(f_lo != t_hi, "overwriting register");
 879       move_regs(f_hi, t_hi);
 880       move_regs(f_lo, t_lo);
 881     } else {
 882       assert(f_hi != t_lo, "overwriting register");
 883       move_regs(f_lo, t_lo);
 884       move_regs(f_hi, t_hi);
 885     }
 886 #endif // LP64
 887 
 888     // special moves from fpu-register to xmm-register
 889     // necessary for method results
 890   } else if (src->is_single_xmm() && !dest->is_single_xmm()) {
 891     __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
 892     __ fld_s(Address(rsp, 0));
 893   } else if (src->is_double_xmm() && !dest->is_double_xmm()) {
 894     __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
 895     __ fld_d(Address(rsp, 0));
 896   } else if (dest->is_single_xmm() && !src->is_single_xmm()) {
 897     __ fstp_s(Address(rsp, 0));
 898     __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
 899   } else if (dest->is_double_xmm() && !src->is_double_xmm()) {
 900     __ fstp_d(Address(rsp, 0));
 901     __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
 902 
 903     // move between xmm-registers
 904   } else if (dest->is_single_xmm()) {
 905     assert(src->is_single_xmm(), "must match");
 906     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 907   } else if (dest->is_double_xmm()) {
 908     assert(src->is_double_xmm(), "must match");
 909     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 910 
 911     // move between fpu-registers (no instruction necessary because of fpu-stack)
 912   } else if (dest->is_single_fpu() || dest->is_double_fpu()) {
 913     assert(src->is_single_fpu() || src->is_double_fpu(), "must match");
 914     assert(src->fpu() == dest->fpu(), "currently should be nothing to do");
 915   } else {
 916     ShouldNotReachHere();
 917   }
 918 }
 919 
 920 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 921   assert(src->is_register(), "should not call otherwise");
 922   assert(dest->is_stack(), "should not call otherwise");
 923 
 924   if (src->is_single_cpu()) {
 925     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 926     if (type == T_OBJECT || type == T_ARRAY) {
 927       __ verify_oop(src->as_register());
 928       __ movptr (dst, src->as_register());
 929     } else {
 930       __ movl (dst, src->as_register());
 931     }
 932 
 933   } else if (src->is_double_cpu()) {
 934     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 935     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 936     __ movptr (dstLO, src->as_register_lo());
 937     NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
 938 
 939   } else if (src->is_single_xmm()) {
 940     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 941     __ movflt(dst_addr, src->as_xmm_float_reg());
 942 
 943   } else if (src->is_double_xmm()) {
 944     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 945     __ movdbl(dst_addr, src->as_xmm_double_reg());
 946 
 947   } else if (src->is_single_fpu()) {
 948     assert(src->fpu_regnr() == 0, "argument must be on TOS");
 949     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 950     if (pop_fpu_stack)     __ fstp_s (dst_addr);
 951     else                   __ fst_s  (dst_addr);
 952 
 953   } else if (src->is_double_fpu()) {
 954     assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
 955     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 956     if (pop_fpu_stack)     __ fstp_d (dst_addr);
 957     else                   __ fst_d  (dst_addr);
 958 
 959   } else {
 960     ShouldNotReachHere();
 961   }
 962 }
 963 
 964 
 965 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 966   LIR_Address* to_addr = dest->as_address_ptr();
 967   PatchingStub* patch = NULL;
 968   Register compressed_src = rscratch1;
 969 
 970   if (type == T_ARRAY || type == T_OBJECT) {
 971     __ verify_oop(src->as_register());
 972 #ifdef _LP64
 973     if (UseCompressedOops && !wide) {
 974       __ movptr(compressed_src, src->as_register());
 975       __ encode_heap_oop(compressed_src);
 976     }
 977 #endif
 978   }
 979 
 980   if (patch_code != lir_patch_none) {
 981     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 982     Address toa = as_Address(to_addr);
 983     assert(toa.disp() != 0, "must have");
 984   }
 985 
 986   int null_check_here = code_offset();
 987   switch (type) {
 988     case T_FLOAT: {
 989       if (src->is_single_xmm()) {
 990         __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
 991       } else {
 992         assert(src->is_single_fpu(), "must be");
 993         assert(src->fpu_regnr() == 0, "argument must be on TOS");
 994         if (pop_fpu_stack)      __ fstp_s(as_Address(to_addr));
 995         else                    __ fst_s (as_Address(to_addr));
 996       }
 997       break;
 998     }
 999 
1000     case T_DOUBLE: {
1001       if (src->is_double_xmm()) {
1002         __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
1003       } else {
1004         assert(src->is_double_fpu(), "must be");
1005         assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
1006         if (pop_fpu_stack)      __ fstp_d(as_Address(to_addr));
1007         else                    __ fst_d (as_Address(to_addr));
1008       }
1009       break;
1010     }
1011 
1012     case T_ARRAY:   // fall through
1013     case T_OBJECT:  // fall through
1014       if (UseCompressedOops && !wide) {
1015         __ movl(as_Address(to_addr), compressed_src);
1016       } else {
1017         __ movptr(as_Address(to_addr), src->as_register());
1018       }
1019       break;
1020     case T_ADDRESS:
1021       __ movptr(as_Address(to_addr), src->as_register());
1022       break;
1023     case T_INT:
1024       __ movl(as_Address(to_addr), src->as_register());
1025       break;
1026 
1027     case T_LONG: {
1028       Register from_lo = src->as_register_lo();
1029       Register from_hi = src->as_register_hi();
1030 #ifdef _LP64
1031       __ movptr(as_Address_lo(to_addr), from_lo);
1032 #else
1033       Register base = to_addr->base()->as_register();
1034       Register index = noreg;
1035       if (to_addr->index()->is_register()) {
1036         index = to_addr->index()->as_register();
1037       }
1038       if (base == from_lo || index == from_lo) {
1039         assert(base != from_hi, "can't be");
1040         assert(index == noreg || (index != base && index != from_hi), "can't handle this");
1041         __ movl(as_Address_hi(to_addr), from_hi);
1042         if (patch != NULL) {
1043           patching_epilog(patch, lir_patch_high, base, info);
1044           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1045           patch_code = lir_patch_low;
1046         }
1047         __ movl(as_Address_lo(to_addr), from_lo);
1048       } else {
1049         assert(index == noreg || (index != base && index != from_lo), "can't handle this");
1050         __ movl(as_Address_lo(to_addr), from_lo);
1051         if (patch != NULL) {
1052           patching_epilog(patch, lir_patch_low, base, info);
1053           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1054           patch_code = lir_patch_high;
1055         }
1056         __ movl(as_Address_hi(to_addr), from_hi);
1057       }
1058 #endif // _LP64
1059       break;
1060     }
1061 
1062     case T_BYTE:    // fall through
1063     case T_BOOLEAN: {
1064       Register src_reg = src->as_register();
1065       Address dst_addr = as_Address(to_addr);
1066       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
1067       __ movb(dst_addr, src_reg);
1068       break;
1069     }
1070 
1071     case T_CHAR:    // fall through
1072     case T_SHORT:
1073       __ movw(as_Address(to_addr), src->as_register());
1074       break;
1075 
1076     default:
1077       ShouldNotReachHere();
1078   }
1079   if (info != NULL) {
1080     add_debug_info_for_null_check(null_check_here, info);
1081   }
1082 
1083   if (patch_code != lir_patch_none) {
1084     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1085   }
1086 }
1087 
1088 
1089 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1090   assert(src->is_stack(), "should not call otherwise");
1091   assert(dest->is_register(), "should not call otherwise");
1092 
1093   if (dest->is_single_cpu()) {
1094     if (type == T_ARRAY || type == T_OBJECT) {
1095       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1096       __ verify_oop(dest->as_register());
1097     } else {
1098       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1099     }
1100 
1101   } else if (dest->is_double_cpu()) {
1102     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1103     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1104     __ movptr(dest->as_register_lo(), src_addr_LO);
1105     NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1106 
1107   } else if (dest->is_single_xmm()) {
1108     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1109     __ movflt(dest->as_xmm_float_reg(), src_addr);
1110 
1111   } else if (dest->is_double_xmm()) {
1112     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1113     __ movdbl(dest->as_xmm_double_reg(), src_addr);
1114 
1115   } else if (dest->is_single_fpu()) {
1116     assert(dest->fpu_regnr() == 0, "dest must be TOS");
1117     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1118     __ fld_s(src_addr);
1119 
1120   } else if (dest->is_double_fpu()) {
1121     assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1122     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1123     __ fld_d(src_addr);
1124 
1125   } else {
1126     ShouldNotReachHere();
1127   }
1128 }
1129 
1130 
1131 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1132   if (src->is_single_stack()) {
1133     if (type == T_OBJECT || type == T_ARRAY) {
1134       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1135       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1136     } else {
1137 #ifndef _LP64
1138       __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1139       __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1140 #else
1141       //no pushl on 64bits
1142       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1143       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1144 #endif
1145     }
1146 
1147   } else if (src->is_double_stack()) {
1148 #ifdef _LP64
1149     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1150     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1151 #else
1152     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1153     // push and pop the part at src + wordSize, adding wordSize for the previous push
1154     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1155     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1156     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1157 #endif // _LP64
1158 
1159   } else {
1160     ShouldNotReachHere();
1161   }
1162 }
1163 
1164 
1165 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
1166   assert(src->is_address(), "should not call otherwise");
1167   assert(dest->is_register(), "should not call otherwise");
1168 
1169   LIR_Address* addr = src->as_address_ptr();
1170   Address from_addr = as_Address(addr);
1171 
1172   switch (type) {
1173     case T_BOOLEAN: // fall through
1174     case T_BYTE:    // fall through
1175     case T_CHAR:    // fall through
1176     case T_SHORT:
1177       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1178         // on pre P6 processors we may get partial register stalls
1179         // so blow away the value of to_rinfo before loading a
1180         // partial word into it.  Do it here so that it precedes
1181         // the potential patch point below.
1182         __ xorptr(dest->as_register(), dest->as_register());
1183       }
1184       break;
1185   }
1186 
1187   PatchingStub* patch = NULL;
1188   if (patch_code != lir_patch_none) {
1189     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1190     assert(from_addr.disp() != 0, "must have");
1191   }
1192   if (info != NULL) {
1193     add_debug_info_for_null_check_here(info);
1194   }
1195 
1196   switch (type) {
1197     case T_FLOAT: {
1198       if (dest->is_single_xmm()) {
1199         __ movflt(dest->as_xmm_float_reg(), from_addr);
1200       } else {
1201         assert(dest->is_single_fpu(), "must be");
1202         assert(dest->fpu_regnr() == 0, "dest must be TOS");
1203         __ fld_s(from_addr);
1204       }
1205       break;
1206     }
1207 
1208     case T_DOUBLE: {
1209       if (dest->is_double_xmm()) {
1210         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1211       } else {
1212         assert(dest->is_double_fpu(), "must be");
1213         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1214         __ fld_d(from_addr);
1215       }
1216       break;
1217     }
1218 
1219     case T_OBJECT:  // fall through
1220     case T_ARRAY:   // fall through
1221       if (UseCompressedOops && !wide) {
1222         __ movl(dest->as_register(), from_addr);
1223       } else {
1224         __ movptr(dest->as_register(), from_addr);
1225       }
1226       break;
1227 
1228     case T_ADDRESS:
1229       __ movptr(dest->as_register(), from_addr);
1230       break;
1231     case T_INT:
1232       __ movl(dest->as_register(), from_addr);
1233       break;
1234 
1235     case T_LONG: {
1236       Register to_lo = dest->as_register_lo();
1237       Register to_hi = dest->as_register_hi();
1238 #ifdef _LP64
1239       __ movptr(to_lo, as_Address_lo(addr));
1240 #else
1241       Register base = addr->base()->as_register();
1242       Register index = noreg;
1243       if (addr->index()->is_register()) {
1244         index = addr->index()->as_register();
1245       }
1246       if ((base == to_lo && index == to_hi) ||
1247           (base == to_hi && index == to_lo)) {
1248         // addresses with 2 registers are only formed as a result of
1249         // array access so this code will never have to deal with
1250         // patches or null checks.
1251         assert(info == NULL && patch == NULL, "must be");
1252         __ lea(to_hi, as_Address(addr));
1253         __ movl(to_lo, Address(to_hi, 0));
1254         __ movl(to_hi, Address(to_hi, BytesPerWord));
1255       } else if (base == to_lo || index == to_lo) {
1256         assert(base != to_hi, "can't be");
1257         assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1258         __ movl(to_hi, as_Address_hi(addr));
1259         if (patch != NULL) {
1260           patching_epilog(patch, lir_patch_high, base, info);
1261           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1262           patch_code = lir_patch_low;
1263         }
1264         __ movl(to_lo, as_Address_lo(addr));
1265       } else {
1266         assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1267         __ movl(to_lo, as_Address_lo(addr));
1268         if (patch != NULL) {
1269           patching_epilog(patch, lir_patch_low, base, info);
1270           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1271           patch_code = lir_patch_high;
1272         }
1273         __ movl(to_hi, as_Address_hi(addr));
1274       }
1275 #endif // _LP64
1276       break;
1277     }
1278 
1279     case T_BOOLEAN: // fall through
1280     case T_BYTE: {
1281       Register dest_reg = dest->as_register();
1282       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1283       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1284         __ movsbl(dest_reg, from_addr);
1285       } else {
1286         __ movb(dest_reg, from_addr);
1287         __ shll(dest_reg, 24);
1288         __ sarl(dest_reg, 24);
1289       }
1290       break;
1291     }
1292 
1293     case T_CHAR: {
1294       Register dest_reg = dest->as_register();
1295       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1296       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1297         __ movzwl(dest_reg, from_addr);
1298       } else {
1299         __ movw(dest_reg, from_addr);
1300       }
1301       break;
1302     }
1303 
1304     case T_SHORT: {
1305       Register dest_reg = dest->as_register();
1306       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1307         __ movswl(dest_reg, from_addr);
1308       } else {
1309         __ movw(dest_reg, from_addr);
1310         __ shll(dest_reg, 16);
1311         __ sarl(dest_reg, 16);
1312       }
1313       break;
1314     }
1315 
1316     default:
1317       ShouldNotReachHere();
1318   }
1319 
1320   if (patch != NULL) {
1321     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1322   }
1323 
1324   if (type == T_ARRAY || type == T_OBJECT) {
1325 #ifdef _LP64
1326     if (UseCompressedOops && !wide) {
1327       __ decode_heap_oop(dest->as_register());
1328     }
1329 #endif
1330     __ verify_oop(dest->as_register());
1331   }
1332 }
1333 
1334 
1335 void LIR_Assembler::prefetchr(LIR_Opr src) {
1336   LIR_Address* addr = src->as_address_ptr();
1337   Address from_addr = as_Address(addr);
1338 
1339   if (VM_Version::supports_sse()) {
1340     switch (ReadPrefetchInstr) {
1341       case 0:
1342         __ prefetchnta(from_addr); break;
1343       case 1:
1344         __ prefetcht0(from_addr); break;
1345       case 2:
1346         __ prefetcht2(from_addr); break;
1347       default:
1348         ShouldNotReachHere(); break;
1349     }
1350   } else if (VM_Version::supports_3dnow_prefetch()) {
1351     __ prefetchr(from_addr);
1352   }
1353 }
1354 
1355 
1356 void LIR_Assembler::prefetchw(LIR_Opr src) {
1357   LIR_Address* addr = src->as_address_ptr();
1358   Address from_addr = as_Address(addr);
1359 
1360   if (VM_Version::supports_sse()) {
1361     switch (AllocatePrefetchInstr) {
1362       case 0:
1363         __ prefetchnta(from_addr); break;
1364       case 1:
1365         __ prefetcht0(from_addr); break;
1366       case 2:
1367         __ prefetcht2(from_addr); break;
1368       case 3:
1369         __ prefetchw(from_addr); break;
1370       default:
1371         ShouldNotReachHere(); break;
1372     }
1373   } else if (VM_Version::supports_3dnow_prefetch()) {
1374     __ prefetchw(from_addr);
1375   }
1376 }
1377 
1378 
1379 NEEDS_CLEANUP; // This could be static?
1380 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1381   int elem_size = type2aelembytes(type);
1382   switch (elem_size) {
1383     case 1: return Address::times_1;
1384     case 2: return Address::times_2;
1385     case 4: return Address::times_4;
1386     case 8: return Address::times_8;
1387   }
1388   ShouldNotReachHere();
1389   return Address::no_scale;
1390 }
1391 
1392 
1393 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1394   switch (op->code()) {
1395     case lir_idiv:
1396     case lir_irem:
1397       arithmetic_idiv(op->code(),
1398                       op->in_opr1(),
1399                       op->in_opr2(),
1400                       op->in_opr3(),
1401                       op->result_opr(),
1402                       op->info());
1403       break;
1404     default:      ShouldNotReachHere(); break;
1405   }
1406 }
1407 
1408 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1409 #ifdef ASSERT
1410   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1411   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1412   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1413 #endif
1414 
1415   if (op->cond() == lir_cond_always) {
1416     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1417     __ jmp (*(op->label()));
1418   } else {
1419     Assembler::Condition acond = Assembler::zero;
1420     if (op->code() == lir_cond_float_branch) {
1421       assert(op->ublock() != NULL, "must have unordered successor");
1422       __ jcc(Assembler::parity, *(op->ublock()->label()));
1423       switch(op->cond()) {
1424         case lir_cond_equal:        acond = Assembler::equal;      break;
1425         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1426         case lir_cond_less:         acond = Assembler::below;      break;
1427         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1428         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1429         case lir_cond_greater:      acond = Assembler::above;      break;
1430         default:                         ShouldNotReachHere();
1431       }
1432     } else {
1433       switch (op->cond()) {
1434         case lir_cond_equal:        acond = Assembler::equal;       break;
1435         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1436         case lir_cond_less:         acond = Assembler::less;        break;
1437         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1438         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1439         case lir_cond_greater:      acond = Assembler::greater;     break;
1440         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1441         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1442         default:                         ShouldNotReachHere();
1443       }
1444     }
1445     __ jcc(acond,*(op->label()));
1446   }
1447 }
1448 
1449 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1450   LIR_Opr src  = op->in_opr();
1451   LIR_Opr dest = op->result_opr();
1452 
1453   switch (op->bytecode()) {
1454     case Bytecodes::_i2l:
1455 #ifdef _LP64
1456       __ movl2ptr(dest->as_register_lo(), src->as_register());
1457 #else
1458       move_regs(src->as_register(), dest->as_register_lo());
1459       move_regs(src->as_register(), dest->as_register_hi());
1460       __ sarl(dest->as_register_hi(), 31);
1461 #endif // LP64
1462       break;
1463 
1464     case Bytecodes::_l2i:
1465       move_regs(src->as_register_lo(), dest->as_register());
1466       break;
1467 
1468     case Bytecodes::_i2b:
1469       move_regs(src->as_register(), dest->as_register());
1470       __ sign_extend_byte(dest->as_register());
1471       break;
1472 
1473     case Bytecodes::_i2c:
1474       move_regs(src->as_register(), dest->as_register());
1475       __ andl(dest->as_register(), 0xFFFF);
1476       break;
1477 
1478     case Bytecodes::_i2s:
1479       move_regs(src->as_register(), dest->as_register());
1480       __ sign_extend_short(dest->as_register());
1481       break;
1482 
1483 
1484     case Bytecodes::_f2d:
1485     case Bytecodes::_d2f:
1486       if (dest->is_single_xmm()) {
1487         __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1488       } else if (dest->is_double_xmm()) {
1489         __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1490       } else {
1491         assert(src->fpu() == dest->fpu(), "register must be equal");
1492         // do nothing (float result is rounded later through spilling)
1493       }
1494       break;
1495 
1496     case Bytecodes::_i2f:
1497     case Bytecodes::_i2d:
1498       if (dest->is_single_xmm()) {
1499         __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1500       } else if (dest->is_double_xmm()) {
1501         __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1502       } else {
1503         assert(dest->fpu() == 0, "result must be on TOS");
1504         __ movl(Address(rsp, 0), src->as_register());
1505         __ fild_s(Address(rsp, 0));
1506       }
1507       break;
1508 
1509     case Bytecodes::_f2i:
1510     case Bytecodes::_d2i:
1511       if (src->is_single_xmm()) {
1512         __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1513       } else if (src->is_double_xmm()) {
1514         __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1515       } else {
1516         assert(src->fpu() == 0, "input must be on TOS");
1517         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
1518         __ fist_s(Address(rsp, 0));
1519         __ movl(dest->as_register(), Address(rsp, 0));
1520         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1521       }
1522 
1523       // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
1524       assert(op->stub() != NULL, "stub required");
1525       __ cmpl(dest->as_register(), 0x80000000);
1526       __ jcc(Assembler::equal, *op->stub()->entry());
1527       __ bind(*op->stub()->continuation());
1528       break;
1529 
1530     case Bytecodes::_l2f:
1531     case Bytecodes::_l2d:
1532       assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)");
1533       assert(dest->fpu() == 0, "result must be on TOS");
1534 
1535       __ movptr(Address(rsp, 0),            src->as_register_lo());
1536       NOT_LP64(__ movl(Address(rsp, BytesPerWord), src->as_register_hi()));
1537       __ fild_d(Address(rsp, 0));
1538       // float result is rounded later through spilling
1539       break;
1540 
1541     case Bytecodes::_f2l:
1542     case Bytecodes::_d2l:
1543       assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)");
1544       assert(src->fpu() == 0, "input must be on TOS");
1545       assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers");
1546 
1547       // instruction sequence too long to inline it here
1548       {
1549         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1550       }
1551       break;
1552 
1553     default: ShouldNotReachHere();
1554   }
1555 }
1556 
1557 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1558   if (op->init_check()) {
1559     __ cmpb(Address(op->klass()->as_register(),
1560                     instanceKlass::init_state_offset()),
1561             instanceKlass::fully_initialized);
1562     add_debug_info_for_null_check_here(op->stub()->info());
1563     __ jcc(Assembler::notEqual, *op->stub()->entry());
1564   }
1565   __ allocate_object(op->obj()->as_register(),
1566                      op->tmp1()->as_register(),
1567                      op->tmp2()->as_register(),
1568                      op->header_size(),
1569                      op->object_size(),
1570                      op->klass()->as_register(),
1571                      *op->stub()->entry());
1572   __ bind(*op->stub()->continuation());
1573 }
1574 
1575 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1576   Register len =  op->len()->as_register();
1577   LP64_ONLY( __ movslq(len, len); )
1578 
1579   if (UseSlowPath ||
1580       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1581       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1582     __ jmp(*op->stub()->entry());
1583   } else {
1584     Register tmp1 = op->tmp1()->as_register();
1585     Register tmp2 = op->tmp2()->as_register();
1586     Register tmp3 = op->tmp3()->as_register();
1587     if (len == tmp1) {
1588       tmp1 = tmp3;
1589     } else if (len == tmp2) {
1590       tmp2 = tmp3;
1591     } else if (len == tmp3) {
1592       // everything is ok
1593     } else {
1594       __ mov(tmp3, len);
1595     }
1596     __ allocate_array(op->obj()->as_register(),
1597                       len,
1598                       tmp1,
1599                       tmp2,
1600                       arrayOopDesc::header_size(op->type()),
1601                       array_element_size(op->type()),
1602                       op->klass()->as_register(),
1603                       *op->stub()->entry());
1604   }
1605   __ bind(*op->stub()->continuation());
1606 }
1607 
1608 void LIR_Assembler::type_profile_helper(Register mdo,
1609                                         ciMethodData *md, ciProfileData *data,
1610                                         Register recv, Label* update_done) {
1611   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1612     Label next_test;
1613     // See if the receiver is receiver[n].
1614     __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1615     __ jccb(Assembler::notEqual, next_test);
1616     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1617     __ addptr(data_addr, DataLayout::counter_increment);
1618     __ jmp(*update_done);
1619     __ bind(next_test);
1620   }
1621 
1622   // Didn't find receiver; find next empty slot and fill it in
1623   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1624     Label next_test;
1625     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1626     __ cmpptr(recv_addr, (intptr_t)NULL_WORD);
1627     __ jccb(Assembler::notEqual, next_test);
1628     __ movptr(recv_addr, recv);
1629     __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1630     __ jmp(*update_done);
1631     __ bind(next_test);
1632   }
1633 }
1634 
1635 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1636   // we always need a stub for the failure case.
1637   CodeStub* stub = op->stub();
1638   Register obj = op->object()->as_register();
1639   Register k_RInfo = op->tmp1()->as_register();
1640   Register klass_RInfo = op->tmp2()->as_register();
1641   Register dst = op->result_opr()->as_register();
1642   ciKlass* k = op->klass();
1643   Register Rtmp1 = noreg;
1644 
1645   // check if it needs to be profiled
1646   ciMethodData* md;
1647   ciProfileData* data;
1648 
1649   if (op->should_profile()) {
1650     ciMethod* method = op->profiled_method();
1651     assert(method != NULL, "Should have method");
1652     int bci = op->profiled_bci();
1653     md = method->method_data_or_null();
1654     assert(md != NULL, "Sanity");
1655     data = md->bci_to_data(bci);
1656     assert(data != NULL,                "need data for type check");
1657     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1658   }
1659   Label profile_cast_success, profile_cast_failure;
1660   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1661   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1662 
1663   if (obj == k_RInfo) {
1664     k_RInfo = dst;
1665   } else if (obj == klass_RInfo) {
1666     klass_RInfo = dst;
1667   }
1668   if (k->is_loaded() && !UseCompressedOops) {
1669     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1670   } else {
1671     Rtmp1 = op->tmp3()->as_register();
1672     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1673   }
1674 
1675   assert_different_registers(obj, k_RInfo, klass_RInfo);
1676   if (!k->is_loaded()) {
1677     jobject2reg_with_patching(k_RInfo, op->info_for_patch());
1678   } else {
1679 #ifdef _LP64
1680     __ movoop(k_RInfo, k->constant_encoding());
1681 #endif // _LP64
1682   }
1683   assert(obj != k_RInfo, "must be different");
1684 
1685   __ cmpptr(obj, (int32_t)NULL_WORD);
1686   if (op->should_profile()) {
1687     Label not_null;
1688     __ jccb(Assembler::notEqual, not_null);
1689     // Object is null; update MDO and exit
1690     Register mdo  = klass_RInfo;
1691     __ movoop(mdo, md->constant_encoding());
1692     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
1693     int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1694     __ orl(data_addr, header_bits);
1695     __ jmp(*obj_is_null);
1696     __ bind(not_null);
1697   } else {
1698     __ jcc(Assembler::equal, *obj_is_null);
1699   }
1700   __ verify_oop(obj);
1701 
1702   if (op->fast_check()) {
1703     // get object class
1704     // not a safepoint as obj null check happens earlier
1705 #ifdef _LP64
1706     if (UseCompressedOops) {
1707       __ load_klass(Rtmp1, obj);
1708       __ cmpptr(k_RInfo, Rtmp1);
1709     } else {
1710       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1711     }
1712 #else
1713     if (k->is_loaded()) {
1714       __ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1715     } else {
1716       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1717     }
1718 #endif
1719     __ jcc(Assembler::notEqual, *failure_target);
1720     // successful cast, fall through to profile or jump
1721   } else {
1722     // get object class
1723     // not a safepoint as obj null check happens earlier
1724     __ load_klass(klass_RInfo, obj);
1725     if (k->is_loaded()) {
1726       // See if we get an immediate positive hit
1727 #ifdef _LP64
1728       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1729 #else
1730       __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1731 #endif // _LP64
1732       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1733         __ jcc(Assembler::notEqual, *failure_target);
1734         // successful cast, fall through to profile or jump
1735       } else {
1736         // See if we get an immediate positive hit
1737         __ jcc(Assembler::equal, *success_target);
1738         // check for self
1739 #ifdef _LP64
1740         __ cmpptr(klass_RInfo, k_RInfo);
1741 #else
1742         __ cmpoop(klass_RInfo, k->constant_encoding());
1743 #endif // _LP64
1744         __ jcc(Assembler::equal, *success_target);
1745 
1746         __ push(klass_RInfo);
1747 #ifdef _LP64
1748         __ push(k_RInfo);
1749 #else
1750         __ pushoop(k->constant_encoding());
1751 #endif // _LP64
1752         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1753         __ pop(klass_RInfo);
1754         __ pop(klass_RInfo);
1755         // result is a boolean
1756         __ cmpl(klass_RInfo, 0);
1757         __ jcc(Assembler::equal, *failure_target);
1758         // successful cast, fall through to profile or jump
1759       }
1760     } else {
1761       // perform the fast part of the checking logic
1762       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1763       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1764       __ push(klass_RInfo);
1765       __ push(k_RInfo);
1766       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1767       __ pop(klass_RInfo);
1768       __ pop(k_RInfo);
1769       // result is a boolean
1770       __ cmpl(k_RInfo, 0);
1771       __ jcc(Assembler::equal, *failure_target);
1772       // successful cast, fall through to profile or jump
1773     }
1774   }
1775   if (op->should_profile()) {
1776     Register mdo  = klass_RInfo, recv = k_RInfo;
1777     __ bind(profile_cast_success);
1778     __ movoop(mdo, md->constant_encoding());
1779     __ load_klass(recv, obj);
1780     Label update_done;
1781     type_profile_helper(mdo, md, data, recv, success);
1782     __ jmp(*success);
1783 
1784     __ bind(profile_cast_failure);
1785     __ movoop(mdo, md->constant_encoding());
1786     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1787     __ subptr(counter_addr, DataLayout::counter_increment);
1788     __ jmp(*failure);
1789   }
1790   __ jmp(*success);
1791 }
1792 
1793 
1794 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1795   LIR_Code code = op->code();
1796   if (code == lir_store_check) {
1797     Register value = op->object()->as_register();
1798     Register array = op->array()->as_register();
1799     Register k_RInfo = op->tmp1()->as_register();
1800     Register klass_RInfo = op->tmp2()->as_register();
1801     Register Rtmp1 = op->tmp3()->as_register();
1802 
1803     CodeStub* stub = op->stub();
1804 
1805     // check if it needs to be profiled
1806     ciMethodData* md;
1807     ciProfileData* data;
1808 
1809     if (op->should_profile()) {
1810       ciMethod* method = op->profiled_method();
1811       assert(method != NULL, "Should have method");
1812       int bci = op->profiled_bci();
1813       md = method->method_data_or_null();
1814       assert(md != NULL, "Sanity");
1815       data = md->bci_to_data(bci);
1816       assert(data != NULL,                "need data for type check");
1817       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1818     }
1819     Label profile_cast_success, profile_cast_failure, done;
1820     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1821     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1822 
1823     __ cmpptr(value, (int32_t)NULL_WORD);
1824     if (op->should_profile()) {
1825       Label not_null;
1826       __ jccb(Assembler::notEqual, not_null);
1827       // Object is null; update MDO and exit
1828       Register mdo  = klass_RInfo;
1829       __ movoop(mdo, md->constant_encoding());
1830       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
1831       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1832       __ orl(data_addr, header_bits);
1833       __ jmp(done);
1834       __ bind(not_null);
1835     } else {
1836       __ jcc(Assembler::equal, done);
1837     }
1838 
1839     add_debug_info_for_null_check_here(op->info_for_exception());
1840     __ load_klass(k_RInfo, array);
1841     __ load_klass(klass_RInfo, value);
1842 
1843     // get instance klass (it's already uncompressed)
1844     __ movptr(k_RInfo, Address(k_RInfo, objArrayKlass::element_klass_offset()));
1845     // perform the fast part of the checking logic
1846     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1847     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1848     __ push(klass_RInfo);
1849     __ push(k_RInfo);
1850     __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1851     __ pop(klass_RInfo);
1852     __ pop(k_RInfo);
1853     // result is a boolean
1854     __ cmpl(k_RInfo, 0);
1855     __ jcc(Assembler::equal, *failure_target);
1856     // fall through to the success case
1857 
1858     if (op->should_profile()) {
1859       Register mdo  = klass_RInfo, recv = k_RInfo;
1860       __ bind(profile_cast_success);
1861       __ movoop(mdo, md->constant_encoding());
1862       __ load_klass(recv, value);
1863       Label update_done;
1864       type_profile_helper(mdo, md, data, recv, &done);
1865       __ jmpb(done);
1866 
1867       __ bind(profile_cast_failure);
1868       __ movoop(mdo, md->constant_encoding());
1869       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1870       __ subptr(counter_addr, DataLayout::counter_increment);
1871       __ jmp(*stub->entry());
1872     }
1873 
1874     __ bind(done);
1875   } else
1876     if (code == lir_checkcast) {
1877       Register obj = op->object()->as_register();
1878       Register dst = op->result_opr()->as_register();
1879       Label success;
1880       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1881       __ bind(success);
1882       if (dst != obj) {
1883         __ mov(dst, obj);
1884       }
1885     } else
1886       if (code == lir_instanceof) {
1887         Register obj = op->object()->as_register();
1888         Register dst = op->result_opr()->as_register();
1889         Label success, failure, done;
1890         emit_typecheck_helper(op, &success, &failure, &failure);
1891         __ bind(failure);
1892         __ xorptr(dst, dst);
1893         __ jmpb(done);
1894         __ bind(success);
1895         __ movptr(dst, 1);
1896         __ bind(done);
1897       } else {
1898         ShouldNotReachHere();
1899       }
1900 
1901 }
1902 
1903 
1904 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1905   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
1906     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1907     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1908     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1909     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1910     Register addr = op->addr()->as_register();
1911     if (os::is_MP()) {
1912       __ lock();
1913     }
1914     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1915 
1916   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1917     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1918     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1919     Register newval = op->new_value()->as_register();
1920     Register cmpval = op->cmp_value()->as_register();
1921     assert(cmpval == rax, "wrong register");
1922     assert(newval != NULL, "new val must be register");
1923     assert(cmpval != newval, "cmp and new values must be in different registers");
1924     assert(cmpval != addr, "cmp and addr must be in different registers");
1925     assert(newval != addr, "new value and addr must be in different registers");
1926 
1927     if ( op->code() == lir_cas_obj) {
1928 #ifdef _LP64
1929       if (UseCompressedOops) {
1930         __ encode_heap_oop(cmpval);
1931         __ mov(rscratch1, newval);
1932         __ encode_heap_oop(rscratch1);
1933         if (os::is_MP()) {
1934           __ lock();
1935         }
1936         // cmpval (rax) is implicitly used by this instruction
1937         __ cmpxchgl(rscratch1, Address(addr, 0));
1938       } else
1939 #endif
1940       {
1941         if (os::is_MP()) {
1942           __ lock();
1943         }
1944         __ cmpxchgptr(newval, Address(addr, 0));
1945       }
1946     } else {
1947       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1948       if (os::is_MP()) {
1949         __ lock();
1950       }
1951       __ cmpxchgl(newval, Address(addr, 0));
1952     }
1953 #ifdef _LP64
1954   } else if (op->code() == lir_cas_long) {
1955     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1956     Register newval = op->new_value()->as_register_lo();
1957     Register cmpval = op->cmp_value()->as_register_lo();
1958     assert(cmpval == rax, "wrong register");
1959     assert(newval != NULL, "new val must be register");
1960     assert(cmpval != newval, "cmp and new values must be in different registers");
1961     assert(cmpval != addr, "cmp and addr must be in different registers");
1962     assert(newval != addr, "new value and addr must be in different registers");
1963     if (os::is_MP()) {
1964       __ lock();
1965     }
1966     __ cmpxchgq(newval, Address(addr, 0));
1967 #endif // _LP64
1968   } else {
1969     Unimplemented();
1970   }
1971 }
1972 
1973 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1974   Assembler::Condition acond, ncond;
1975   switch (condition) {
1976     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1977     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1978     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1979     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1980     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1981     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1982     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1983     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1984     default:                    ShouldNotReachHere();
1985   }
1986 
1987   if (opr1->is_cpu_register()) {
1988     reg2reg(opr1, result);
1989   } else if (opr1->is_stack()) {
1990     stack2reg(opr1, result, result->type());
1991   } else if (opr1->is_constant()) {
1992     const2reg(opr1, result, lir_patch_none, NULL);
1993   } else {
1994     ShouldNotReachHere();
1995   }
1996 
1997   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1998     // optimized version that does not require a branch
1999     if (opr2->is_single_cpu()) {
2000       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
2001       __ cmov(ncond, result->as_register(), opr2->as_register());
2002     } else if (opr2->is_double_cpu()) {
2003       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2004       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2005       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
2006       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
2007     } else if (opr2->is_single_stack()) {
2008       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
2009     } else if (opr2->is_double_stack()) {
2010       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
2011       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
2012     } else {
2013       ShouldNotReachHere();
2014     }
2015 
2016   } else {
2017     Label skip;
2018     __ jcc (acond, skip);
2019     if (opr2->is_cpu_register()) {
2020       reg2reg(opr2, result);
2021     } else if (opr2->is_stack()) {
2022       stack2reg(opr2, result, result->type());
2023     } else if (opr2->is_constant()) {
2024       const2reg(opr2, result, lir_patch_none, NULL);
2025     } else {
2026       ShouldNotReachHere();
2027     }
2028     __ bind(skip);
2029   }
2030 }
2031 
2032 
2033 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
2034   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
2035 
2036   if (left->is_single_cpu()) {
2037     assert(left == dest, "left and dest must be equal");
2038     Register lreg = left->as_register();
2039 
2040     if (right->is_single_cpu()) {
2041       // cpu register - cpu register
2042       Register rreg = right->as_register();
2043       switch (code) {
2044         case lir_add: __ addl (lreg, rreg); break;
2045         case lir_sub: __ subl (lreg, rreg); break;
2046         case lir_mul: __ imull(lreg, rreg); break;
2047         default:      ShouldNotReachHere();
2048       }
2049 
2050     } else if (right->is_stack()) {
2051       // cpu register - stack
2052       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2053       switch (code) {
2054         case lir_add: __ addl(lreg, raddr); break;
2055         case lir_sub: __ subl(lreg, raddr); break;
2056         default:      ShouldNotReachHere();
2057       }
2058 
2059     } else if (right->is_constant()) {
2060       // cpu register - constant
2061       jint c = right->as_constant_ptr()->as_jint();
2062       switch (code) {
2063         case lir_add: {
2064           __ incrementl(lreg, c);
2065           break;
2066         }
2067         case lir_sub: {
2068           __ decrementl(lreg, c);
2069           break;
2070         }
2071         default: ShouldNotReachHere();
2072       }
2073 
2074     } else {
2075       ShouldNotReachHere();
2076     }
2077 
2078   } else if (left->is_double_cpu()) {
2079     assert(left == dest, "left and dest must be equal");
2080     Register lreg_lo = left->as_register_lo();
2081     Register lreg_hi = left->as_register_hi();
2082 
2083     if (right->is_double_cpu()) {
2084       // cpu register - cpu register
2085       Register rreg_lo = right->as_register_lo();
2086       Register rreg_hi = right->as_register_hi();
2087       NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
2088       LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
2089       switch (code) {
2090         case lir_add:
2091           __ addptr(lreg_lo, rreg_lo);
2092           NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2093           break;
2094         case lir_sub:
2095           __ subptr(lreg_lo, rreg_lo);
2096           NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2097           break;
2098         case lir_mul:
2099 #ifdef _LP64
2100           __ imulq(lreg_lo, rreg_lo);
2101 #else
2102           assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2103           __ imull(lreg_hi, rreg_lo);
2104           __ imull(rreg_hi, lreg_lo);
2105           __ addl (rreg_hi, lreg_hi);
2106           __ mull (rreg_lo);
2107           __ addl (lreg_hi, rreg_hi);
2108 #endif // _LP64
2109           break;
2110         default:
2111           ShouldNotReachHere();
2112       }
2113 
2114     } else if (right->is_constant()) {
2115       // cpu register - constant
2116 #ifdef _LP64
2117       jlong c = right->as_constant_ptr()->as_jlong_bits();
2118       __ movptr(r10, (intptr_t) c);
2119       switch (code) {
2120         case lir_add:
2121           __ addptr(lreg_lo, r10);
2122           break;
2123         case lir_sub:
2124           __ subptr(lreg_lo, r10);
2125           break;
2126         default:
2127           ShouldNotReachHere();
2128       }
2129 #else
2130       jint c_lo = right->as_constant_ptr()->as_jint_lo();
2131       jint c_hi = right->as_constant_ptr()->as_jint_hi();
2132       switch (code) {
2133         case lir_add:
2134           __ addptr(lreg_lo, c_lo);
2135           __ adcl(lreg_hi, c_hi);
2136           break;
2137         case lir_sub:
2138           __ subptr(lreg_lo, c_lo);
2139           __ sbbl(lreg_hi, c_hi);
2140           break;
2141         default:
2142           ShouldNotReachHere();
2143       }
2144 #endif // _LP64
2145 
2146     } else {
2147       ShouldNotReachHere();
2148     }
2149 
2150   } else if (left->is_single_xmm()) {
2151     assert(left == dest, "left and dest must be equal");
2152     XMMRegister lreg = left->as_xmm_float_reg();
2153 
2154     if (right->is_single_xmm()) {
2155       XMMRegister rreg = right->as_xmm_float_reg();
2156       switch (code) {
2157         case lir_add: __ addss(lreg, rreg);  break;
2158         case lir_sub: __ subss(lreg, rreg);  break;
2159         case lir_mul_strictfp: // fall through
2160         case lir_mul: __ mulss(lreg, rreg);  break;
2161         case lir_div_strictfp: // fall through
2162         case lir_div: __ divss(lreg, rreg);  break;
2163         default: ShouldNotReachHere();
2164       }
2165     } else {
2166       Address raddr;
2167       if (right->is_single_stack()) {
2168         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2169       } else if (right->is_constant()) {
2170         // hack for now
2171         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2172       } else {
2173         ShouldNotReachHere();
2174       }
2175       switch (code) {
2176         case lir_add: __ addss(lreg, raddr);  break;
2177         case lir_sub: __ subss(lreg, raddr);  break;
2178         case lir_mul_strictfp: // fall through
2179         case lir_mul: __ mulss(lreg, raddr);  break;
2180         case lir_div_strictfp: // fall through
2181         case lir_div: __ divss(lreg, raddr);  break;
2182         default: ShouldNotReachHere();
2183       }
2184     }
2185 
2186   } else if (left->is_double_xmm()) {
2187     assert(left == dest, "left and dest must be equal");
2188 
2189     XMMRegister lreg = left->as_xmm_double_reg();
2190     if (right->is_double_xmm()) {
2191       XMMRegister rreg = right->as_xmm_double_reg();
2192       switch (code) {
2193         case lir_add: __ addsd(lreg, rreg);  break;
2194         case lir_sub: __ subsd(lreg, rreg);  break;
2195         case lir_mul_strictfp: // fall through
2196         case lir_mul: __ mulsd(lreg, rreg);  break;
2197         case lir_div_strictfp: // fall through
2198         case lir_div: __ divsd(lreg, rreg);  break;
2199         default: ShouldNotReachHere();
2200       }
2201     } else {
2202       Address raddr;
2203       if (right->is_double_stack()) {
2204         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2205       } else if (right->is_constant()) {
2206         // hack for now
2207         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2208       } else {
2209         ShouldNotReachHere();
2210       }
2211       switch (code) {
2212         case lir_add: __ addsd(lreg, raddr);  break;
2213         case lir_sub: __ subsd(lreg, raddr);  break;
2214         case lir_mul_strictfp: // fall through
2215         case lir_mul: __ mulsd(lreg, raddr);  break;
2216         case lir_div_strictfp: // fall through
2217         case lir_div: __ divsd(lreg, raddr);  break;
2218         default: ShouldNotReachHere();
2219       }
2220     }
2221 
2222   } else if (left->is_single_fpu()) {
2223     assert(dest->is_single_fpu(),  "fpu stack allocation required");
2224 
2225     if (right->is_single_fpu()) {
2226       arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack);
2227 
2228     } else {
2229       assert(left->fpu_regnr() == 0, "left must be on TOS");
2230       assert(dest->fpu_regnr() == 0, "dest must be on TOS");
2231 
2232       Address raddr;
2233       if (right->is_single_stack()) {
2234         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2235       } else if (right->is_constant()) {
2236         address const_addr = float_constant(right->as_jfloat());
2237         assert(const_addr != NULL, "incorrect float/double constant maintainance");
2238         // hack for now
2239         raddr = __ as_Address(InternalAddress(const_addr));
2240       } else {
2241         ShouldNotReachHere();
2242       }
2243 
2244       switch (code) {
2245         case lir_add: __ fadd_s(raddr); break;
2246         case lir_sub: __ fsub_s(raddr); break;
2247         case lir_mul_strictfp: // fall through
2248         case lir_mul: __ fmul_s(raddr); break;
2249         case lir_div_strictfp: // fall through
2250         case lir_div: __ fdiv_s(raddr); break;
2251         default:      ShouldNotReachHere();
2252       }
2253     }
2254 
2255   } else if (left->is_double_fpu()) {
2256     assert(dest->is_double_fpu(),  "fpu stack allocation required");
2257 
2258     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2259       // Double values require special handling for strictfp mul/div on x86
2260       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
2261       __ fmulp(left->fpu_regnrLo() + 1);
2262     }
2263 
2264     if (right->is_double_fpu()) {
2265       arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack);
2266 
2267     } else {
2268       assert(left->fpu_regnrLo() == 0, "left must be on TOS");
2269       assert(dest->fpu_regnrLo() == 0, "dest must be on TOS");
2270 
2271       Address raddr;
2272       if (right->is_double_stack()) {
2273         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2274       } else if (right->is_constant()) {
2275         // hack for now
2276         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2277       } else {
2278         ShouldNotReachHere();
2279       }
2280 
2281       switch (code) {
2282         case lir_add: __ fadd_d(raddr); break;
2283         case lir_sub: __ fsub_d(raddr); break;
2284         case lir_mul_strictfp: // fall through
2285         case lir_mul: __ fmul_d(raddr); break;
2286         case lir_div_strictfp: // fall through
2287         case lir_div: __ fdiv_d(raddr); break;
2288         default: ShouldNotReachHere();
2289       }
2290     }
2291 
2292     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2293       // Double values require special handling for strictfp mul/div on x86
2294       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
2295       __ fmulp(dest->fpu_regnrLo() + 1);
2296     }
2297 
2298   } else if (left->is_single_stack() || left->is_address()) {
2299     assert(left == dest, "left and dest must be equal");
2300 
2301     Address laddr;
2302     if (left->is_single_stack()) {
2303       laddr = frame_map()->address_for_slot(left->single_stack_ix());
2304     } else if (left->is_address()) {
2305       laddr = as_Address(left->as_address_ptr());
2306     } else {
2307       ShouldNotReachHere();
2308     }
2309 
2310     if (right->is_single_cpu()) {
2311       Register rreg = right->as_register();
2312       switch (code) {
2313         case lir_add: __ addl(laddr, rreg); break;
2314         case lir_sub: __ subl(laddr, rreg); break;
2315         default:      ShouldNotReachHere();
2316       }
2317     } else if (right->is_constant()) {
2318       jint c = right->as_constant_ptr()->as_jint();
2319       switch (code) {
2320         case lir_add: {
2321           __ incrementl(laddr, c);
2322           break;
2323         }
2324         case lir_sub: {
2325           __ decrementl(laddr, c);
2326           break;
2327         }
2328         default: ShouldNotReachHere();
2329       }
2330     } else {
2331       ShouldNotReachHere();
2332     }
2333 
2334   } else {
2335     ShouldNotReachHere();
2336   }
2337 }
2338 
2339 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) {
2340   assert(pop_fpu_stack  || (left_index     == dest_index || right_index     == dest_index), "invalid LIR");
2341   assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR");
2342   assert(left_index == 0 || right_index == 0, "either must be on top of stack");
2343 
2344   bool left_is_tos = (left_index == 0);
2345   bool dest_is_tos = (dest_index == 0);
2346   int non_tos_index = (left_is_tos ? right_index : left_index);
2347 
2348   switch (code) {
2349     case lir_add:
2350       if (pop_fpu_stack)       __ faddp(non_tos_index);
2351       else if (dest_is_tos)    __ fadd (non_tos_index);
2352       else                     __ fadda(non_tos_index);
2353       break;
2354 
2355     case lir_sub:
2356       if (left_is_tos) {
2357         if (pop_fpu_stack)     __ fsubrp(non_tos_index);
2358         else if (dest_is_tos)  __ fsub  (non_tos_index);
2359         else                   __ fsubra(non_tos_index);
2360       } else {
2361         if (pop_fpu_stack)     __ fsubp (non_tos_index);
2362         else if (dest_is_tos)  __ fsubr (non_tos_index);
2363         else                   __ fsuba (non_tos_index);
2364       }
2365       break;
2366 
2367     case lir_mul_strictfp: // fall through
2368     case lir_mul:
2369       if (pop_fpu_stack)       __ fmulp(non_tos_index);
2370       else if (dest_is_tos)    __ fmul (non_tos_index);
2371       else                     __ fmula(non_tos_index);
2372       break;
2373 
2374     case lir_div_strictfp: // fall through
2375     case lir_div:
2376       if (left_is_tos) {
2377         if (pop_fpu_stack)     __ fdivrp(non_tos_index);
2378         else if (dest_is_tos)  __ fdiv  (non_tos_index);
2379         else                   __ fdivra(non_tos_index);
2380       } else {
2381         if (pop_fpu_stack)     __ fdivp (non_tos_index);
2382         else if (dest_is_tos)  __ fdivr (non_tos_index);
2383         else                   __ fdiva (non_tos_index);
2384       }
2385       break;
2386 
2387     case lir_rem:
2388       assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation");
2389       __ fremr(noreg);
2390       break;
2391 
2392     default:
2393       ShouldNotReachHere();
2394   }
2395 }
2396 
2397 
2398 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
2399   if (value->is_double_xmm()) {
2400     switch(code) {
2401       case lir_abs :
2402         {
2403           if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2404             __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2405           }
2406           __ andpd(dest->as_xmm_double_reg(),
2407                     ExternalAddress((address)double_signmask_pool));
2408         }
2409         break;
2410 
2411       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2412       // all other intrinsics are not available in the SSE instruction set, so FPU is used
2413       default      : ShouldNotReachHere();
2414     }
2415 
2416   } else if (value->is_double_fpu()) {
2417     assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2418     switch(code) {
2419       case lir_log   : __ flog() ; break;
2420       case lir_log10 : __ flog10() ; break;
2421       case lir_abs   : __ fabs() ; break;
2422       case lir_sqrt  : __ fsqrt(); break;
2423       case lir_sin   :
2424         // Should consider not saving rbx, if not necessary
2425         __ trigfunc('s', op->as_Op2()->fpu_stack_size());
2426         break;
2427       case lir_cos :
2428         // Should consider not saving rbx, if not necessary
2429         assert(op->as_Op2()->fpu_stack_size() <= 6, "sin and cos need two free stack slots");
2430         __ trigfunc('c', op->as_Op2()->fpu_stack_size());
2431         break;
2432       case lir_tan :
2433         // Should consider not saving rbx, if not necessary
2434         __ trigfunc('t', op->as_Op2()->fpu_stack_size());
2435         break;
2436       default      : ShouldNotReachHere();
2437     }
2438   } else {
2439     Unimplemented();
2440   }
2441 }
2442 
2443 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2444   // assert(left->destroys_register(), "check");
2445   if (left->is_single_cpu()) {
2446     Register reg = left->as_register();
2447     if (right->is_constant()) {
2448       int val = right->as_constant_ptr()->as_jint();
2449       switch (code) {
2450         case lir_logic_and: __ andl (reg, val); break;
2451         case lir_logic_or:  __ orl  (reg, val); break;
2452         case lir_logic_xor: __ xorl (reg, val); break;
2453         default: ShouldNotReachHere();
2454       }
2455     } else if (right->is_stack()) {
2456       // added support for stack operands
2457       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2458       switch (code) {
2459         case lir_logic_and: __ andl (reg, raddr); break;
2460         case lir_logic_or:  __ orl  (reg, raddr); break;
2461         case lir_logic_xor: __ xorl (reg, raddr); break;
2462         default: ShouldNotReachHere();
2463       }
2464     } else {
2465       Register rright = right->as_register();
2466       switch (code) {
2467         case lir_logic_and: __ andptr (reg, rright); break;
2468         case lir_logic_or : __ orptr  (reg, rright); break;
2469         case lir_logic_xor: __ xorptr (reg, rright); break;
2470         default: ShouldNotReachHere();
2471       }
2472     }
2473     move_regs(reg, dst->as_register());
2474   } else {
2475     Register l_lo = left->as_register_lo();
2476     Register l_hi = left->as_register_hi();
2477     if (right->is_constant()) {
2478 #ifdef _LP64
2479       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2480       switch (code) {
2481         case lir_logic_and:
2482           __ andq(l_lo, rscratch1);
2483           break;
2484         case lir_logic_or:
2485           __ orq(l_lo, rscratch1);
2486           break;
2487         case lir_logic_xor:
2488           __ xorq(l_lo, rscratch1);
2489           break;
2490         default: ShouldNotReachHere();
2491       }
2492 #else
2493       int r_lo = right->as_constant_ptr()->as_jint_lo();
2494       int r_hi = right->as_constant_ptr()->as_jint_hi();
2495       switch (code) {
2496         case lir_logic_and:
2497           __ andl(l_lo, r_lo);
2498           __ andl(l_hi, r_hi);
2499           break;
2500         case lir_logic_or:
2501           __ orl(l_lo, r_lo);
2502           __ orl(l_hi, r_hi);
2503           break;
2504         case lir_logic_xor:
2505           __ xorl(l_lo, r_lo);
2506           __ xorl(l_hi, r_hi);
2507           break;
2508         default: ShouldNotReachHere();
2509       }
2510 #endif // _LP64
2511     } else {
2512 #ifdef _LP64
2513       Register r_lo;
2514       if (right->type() == T_OBJECT || right->type() == T_ARRAY) {
2515         r_lo = right->as_register();
2516       } else {
2517         r_lo = right->as_register_lo();
2518       }
2519 #else
2520       Register r_lo = right->as_register_lo();
2521       Register r_hi = right->as_register_hi();
2522       assert(l_lo != r_hi, "overwriting registers");
2523 #endif
2524       switch (code) {
2525         case lir_logic_and:
2526           __ andptr(l_lo, r_lo);
2527           NOT_LP64(__ andptr(l_hi, r_hi);)
2528           break;
2529         case lir_logic_or:
2530           __ orptr(l_lo, r_lo);
2531           NOT_LP64(__ orptr(l_hi, r_hi);)
2532           break;
2533         case lir_logic_xor:
2534           __ xorptr(l_lo, r_lo);
2535           NOT_LP64(__ xorptr(l_hi, r_hi);)
2536           break;
2537         default: ShouldNotReachHere();
2538       }
2539     }
2540 
2541     Register dst_lo = dst->as_register_lo();
2542     Register dst_hi = dst->as_register_hi();
2543 
2544 #ifdef _LP64
2545     move_regs(l_lo, dst_lo);
2546 #else
2547     if (dst_lo == l_hi) {
2548       assert(dst_hi != l_lo, "overwriting registers");
2549       move_regs(l_hi, dst_hi);
2550       move_regs(l_lo, dst_lo);
2551     } else {
2552       assert(dst_lo != l_hi, "overwriting registers");
2553       move_regs(l_lo, dst_lo);
2554       move_regs(l_hi, dst_hi);
2555     }
2556 #endif // _LP64
2557   }
2558 }
2559 
2560 
2561 // we assume that rax, and rdx can be overwritten
2562 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2563 
2564   assert(left->is_single_cpu(),   "left must be register");
2565   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2566   assert(result->is_single_cpu(), "result must be register");
2567 
2568   //  assert(left->destroys_register(), "check");
2569   //  assert(right->destroys_register(), "check");
2570 
2571   Register lreg = left->as_register();
2572   Register dreg = result->as_register();
2573 
2574   if (right->is_constant()) {
2575     int divisor = right->as_constant_ptr()->as_jint();
2576     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2577     if (code == lir_idiv) {
2578       assert(lreg == rax, "must be rax,");
2579       assert(temp->as_register() == rdx, "tmp register must be rdx");
2580       __ cdql(); // sign extend into rdx:rax
2581       if (divisor == 2) {
2582         __ subl(lreg, rdx);
2583       } else {
2584         __ andl(rdx, divisor - 1);
2585         __ addl(lreg, rdx);
2586       }
2587       __ sarl(lreg, log2_intptr(divisor));
2588       move_regs(lreg, dreg);
2589     } else if (code == lir_irem) {
2590       Label done;
2591       __ mov(dreg, lreg);
2592       __ andl(dreg, 0x80000000 | (divisor - 1));
2593       __ jcc(Assembler::positive, done);
2594       __ decrement(dreg);
2595       __ orl(dreg, ~(divisor - 1));
2596       __ increment(dreg);
2597       __ bind(done);
2598     } else {
2599       ShouldNotReachHere();
2600     }
2601   } else {
2602     Register rreg = right->as_register();
2603     assert(lreg == rax, "left register must be rax,");
2604     assert(rreg != rdx, "right register must not be rdx");
2605     assert(temp->as_register() == rdx, "tmp register must be rdx");
2606 
2607     move_regs(lreg, rax);
2608 
2609     int idivl_offset = __ corrected_idivl(rreg);
2610     add_debug_info_for_div0(idivl_offset, info);
2611     if (code == lir_irem) {
2612       move_regs(rdx, dreg); // result is in rdx
2613     } else {
2614       move_regs(rax, dreg);
2615     }
2616   }
2617 }
2618 
2619 
2620 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2621   if (opr1->is_single_cpu()) {
2622     Register reg1 = opr1->as_register();
2623     if (opr2->is_single_cpu()) {
2624       // cpu register - cpu register
2625       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2626         __ cmpptr(reg1, opr2->as_register());
2627       } else {
2628         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
2629         __ cmpl(reg1, opr2->as_register());
2630       }
2631     } else if (opr2->is_stack()) {
2632       // cpu register - stack
2633       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2634         __ cmpptr(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2635       } else {
2636         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2637       }
2638     } else if (opr2->is_constant()) {
2639       // cpu register - constant
2640       LIR_Const* c = opr2->as_constant_ptr();
2641       if (c->type() == T_INT) {
2642         __ cmpl(reg1, c->as_jint());
2643       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2644         // In 64bit oops are single register
2645         jobject o = c->as_jobject();
2646         if (o == NULL) {
2647           __ cmpptr(reg1, (int32_t)NULL_WORD);
2648         } else {
2649 #ifdef _LP64
2650           __ movoop(rscratch1, o);
2651           __ cmpptr(reg1, rscratch1);
2652 #else
2653           __ cmpoop(reg1, c->as_jobject());
2654 #endif // _LP64
2655         }
2656       } else {
2657         ShouldNotReachHere();
2658       }
2659       // cpu register - address
2660     } else if (opr2->is_address()) {
2661       if (op->info() != NULL) {
2662         add_debug_info_for_null_check_here(op->info());
2663       }
2664       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2665     } else {
2666       ShouldNotReachHere();
2667     }
2668 
2669   } else if(opr1->is_double_cpu()) {
2670     Register xlo = opr1->as_register_lo();
2671     Register xhi = opr1->as_register_hi();
2672     if (opr2->is_double_cpu()) {
2673 #ifdef _LP64
2674       __ cmpptr(xlo, opr2->as_register_lo());
2675 #else
2676       // cpu register - cpu register
2677       Register ylo = opr2->as_register_lo();
2678       Register yhi = opr2->as_register_hi();
2679       __ subl(xlo, ylo);
2680       __ sbbl(xhi, yhi);
2681       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2682         __ orl(xhi, xlo);
2683       }
2684 #endif // _LP64
2685     } else if (opr2->is_constant()) {
2686       // cpu register - constant 0
2687       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2688 #ifdef _LP64
2689       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2690 #else
2691       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2692       __ orl(xhi, xlo);
2693 #endif // _LP64
2694     } else {
2695       ShouldNotReachHere();
2696     }
2697 
2698   } else if (opr1->is_single_xmm()) {
2699     XMMRegister reg1 = opr1->as_xmm_float_reg();
2700     if (opr2->is_single_xmm()) {
2701       // xmm register - xmm register
2702       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2703     } else if (opr2->is_stack()) {
2704       // xmm register - stack
2705       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2706     } else if (opr2->is_constant()) {
2707       // xmm register - constant
2708       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2709     } else if (opr2->is_address()) {
2710       // xmm register - address
2711       if (op->info() != NULL) {
2712         add_debug_info_for_null_check_here(op->info());
2713       }
2714       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2715     } else {
2716       ShouldNotReachHere();
2717     }
2718 
2719   } else if (opr1->is_double_xmm()) {
2720     XMMRegister reg1 = opr1->as_xmm_double_reg();
2721     if (opr2->is_double_xmm()) {
2722       // xmm register - xmm register
2723       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2724     } else if (opr2->is_stack()) {
2725       // xmm register - stack
2726       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2727     } else if (opr2->is_constant()) {
2728       // xmm register - constant
2729       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2730     } else if (opr2->is_address()) {
2731       // xmm register - address
2732       if (op->info() != NULL) {
2733         add_debug_info_for_null_check_here(op->info());
2734       }
2735       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2736     } else {
2737       ShouldNotReachHere();
2738     }
2739 
2740   } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) {
2741     assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)");
2742     assert(opr2->is_fpu_register(), "both must be registers");
2743     __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2744 
2745   } else if (opr1->is_address() && opr2->is_constant()) {
2746     LIR_Const* c = opr2->as_constant_ptr();
2747 #ifdef _LP64
2748     if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2749       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2750       __ movoop(rscratch1, c->as_jobject());
2751     }
2752 #endif // LP64
2753     if (op->info() != NULL) {
2754       add_debug_info_for_null_check_here(op->info());
2755     }
2756     // special case: address - constant
2757     LIR_Address* addr = opr1->as_address_ptr();
2758     if (c->type() == T_INT) {
2759       __ cmpl(as_Address(addr), c->as_jint());
2760     } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2761 #ifdef _LP64
2762       // %%% Make this explode if addr isn't reachable until we figure out a
2763       // better strategy by giving noreg as the temp for as_Address
2764       __ cmpptr(rscratch1, as_Address(addr, noreg));
2765 #else
2766       __ cmpoop(as_Address(addr), c->as_jobject());
2767 #endif // _LP64
2768     } else {
2769       ShouldNotReachHere();
2770     }
2771 
2772   } else {
2773     ShouldNotReachHere();
2774   }
2775 }
2776 
2777 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2778   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2779     if (left->is_single_xmm()) {
2780       assert(right->is_single_xmm(), "must match");
2781       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2782     } else if (left->is_double_xmm()) {
2783       assert(right->is_double_xmm(), "must match");
2784       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2785 
2786     } else {
2787       assert(left->is_single_fpu() || left->is_double_fpu(), "must be");
2788       assert(right->is_single_fpu() || right->is_double_fpu(), "must match");
2789 
2790       assert(left->fpu() == 0, "left must be on TOS");
2791       __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2792                   op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2793     }
2794   } else {
2795     assert(code == lir_cmp_l2i, "check");
2796 #ifdef _LP64
2797     Label done;
2798     Register dest = dst->as_register();
2799     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2800     __ movl(dest, -1);
2801     __ jccb(Assembler::less, done);
2802     __ set_byte_if_not_zero(dest);
2803     __ movzbl(dest, dest);
2804     __ bind(done);
2805 #else
2806     __ lcmp2int(left->as_register_hi(),
2807                 left->as_register_lo(),
2808                 right->as_register_hi(),
2809                 right->as_register_lo());
2810     move_regs(left->as_register_hi(), dst->as_register());
2811 #endif // _LP64
2812   }
2813 }
2814 
2815 
2816 void LIR_Assembler::align_call(LIR_Code code) {
2817   if (os::is_MP()) {
2818     // make sure that the displacement word of the call ends up word aligned
2819     int offset = __ offset();
2820     switch (code) {
2821       case lir_static_call:
2822       case lir_optvirtual_call:
2823       case lir_dynamic_call:
2824         offset += NativeCall::displacement_offset;
2825         break;
2826       case lir_icvirtual_call:
2827         offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
2828       break;
2829       case lir_virtual_call:  // currently, sparc-specific for niagara
2830       default: ShouldNotReachHere();
2831     }
2832     while (offset++ % BytesPerWord != 0) {
2833       __ nop();
2834     }
2835   }
2836 }
2837 
2838 
2839 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2840   assert(!os::is_MP() || (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2841          "must be aligned");
2842   __ call(AddressLiteral(op->addr(), rtype));
2843   add_call_info(code_offset(), op->info());
2844 }
2845 
2846 
2847 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2848   RelocationHolder rh = virtual_call_Relocation::spec(pc());
2849   __ movoop(IC_Klass, (jobject)Universe::non_oop_word());
2850   assert(!os::is_MP() ||
2851          (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2852          "must be aligned");
2853   __ call(AddressLiteral(op->addr(), rh));
2854   add_call_info(code_offset(), op->info());
2855 }
2856 
2857 
2858 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2859 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2860   ShouldNotReachHere();
2861 }
2862 
2863 
2864 void LIR_Assembler::emit_static_call_stub() {
2865   address call_pc = __ pc();
2866   address stub = __ start_a_stub(call_stub_size);
2867   if (stub == NULL) {
2868     bailout("static call stub overflow");
2869     return;
2870   }
2871 
2872   int start = __ offset();
2873   if (os::is_MP()) {
2874     // make sure that the displacement word of the call ends up word aligned
2875     int offset = __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset;
2876     while (offset++ % BytesPerWord != 0) {
2877       __ nop();
2878     }
2879   }
2880   __ relocate(static_stub_Relocation::spec(call_pc));
2881   __ movoop(rbx, (jobject)NULL);
2882   // must be set to -1 at code generation time
2883   assert(!os::is_MP() || ((__ offset() + 1) % BytesPerWord) == 0, "must be aligned on MP");
2884   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2885   __ jump(RuntimeAddress(__ pc()));
2886 
2887   assert(__ offset() - start <= call_stub_size, "stub too big");
2888   __ end_a_stub();
2889 }
2890 
2891 
2892 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2893   assert(exceptionOop->as_register() == rax, "must match");
2894   assert(exceptionPC->as_register() == rdx, "must match");
2895 
2896   // exception object is not added to oop map by LinearScan
2897   // (LinearScan assumes that no oops are in fixed registers)
2898   info->add_register_oop(exceptionOop);
2899   Runtime1::StubID unwind_id;
2900 
2901   // get current pc information
2902   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2903   int pc_for_athrow_offset = __ offset();
2904   InternalAddress pc_for_athrow(__ pc());
2905   __ lea(exceptionPC->as_register(), pc_for_athrow);
2906   add_call_info(pc_for_athrow_offset, info); // for exception handler
2907 
2908   __ verify_not_null_oop(rax);
2909   // search an exception handler (rax: exception oop, rdx: throwing pc)
2910   if (compilation()->has_fpu_code()) {
2911     unwind_id = Runtime1::handle_exception_id;
2912   } else {
2913     unwind_id = Runtime1::handle_exception_nofpu_id;
2914   }
2915   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2916 
2917   // enough room for two byte trap
2918   __ nop();
2919 }
2920 
2921 
2922 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2923   assert(exceptionOop->as_register() == rax, "must match");
2924 
2925   __ jmp(_unwind_handler_entry);
2926 }
2927 
2928 
2929 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2930 
2931   // optimized version for linear scan:
2932   // * count must be already in ECX (guaranteed by LinearScan)
2933   // * left and dest must be equal
2934   // * tmp must be unused
2935   assert(count->as_register() == SHIFT_count, "count must be in ECX");
2936   assert(left == dest, "left and dest must be equal");
2937   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2938 
2939   if (left->is_single_cpu()) {
2940     Register value = left->as_register();
2941     assert(value != SHIFT_count, "left cannot be ECX");
2942 
2943     switch (code) {
2944       case lir_shl:  __ shll(value); break;
2945       case lir_shr:  __ sarl(value); break;
2946       case lir_ushr: __ shrl(value); break;
2947       default: ShouldNotReachHere();
2948     }
2949   } else if (left->is_double_cpu()) {
2950     Register lo = left->as_register_lo();
2951     Register hi = left->as_register_hi();
2952     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2953 #ifdef _LP64
2954     switch (code) {
2955       case lir_shl:  __ shlptr(lo);        break;
2956       case lir_shr:  __ sarptr(lo);        break;
2957       case lir_ushr: __ shrptr(lo);        break;
2958       default: ShouldNotReachHere();
2959     }
2960 #else
2961 
2962     switch (code) {
2963       case lir_shl:  __ lshl(hi, lo);        break;
2964       case lir_shr:  __ lshr(hi, lo, true);  break;
2965       case lir_ushr: __ lshr(hi, lo, false); break;
2966       default: ShouldNotReachHere();
2967     }
2968 #endif // LP64
2969   } else {
2970     ShouldNotReachHere();
2971   }
2972 }
2973 
2974 
2975 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2976   if (dest->is_single_cpu()) {
2977     // first move left into dest so that left is not destroyed by the shift
2978     Register value = dest->as_register();
2979     count = count & 0x1F; // Java spec
2980 
2981     move_regs(left->as_register(), value);
2982     switch (code) {
2983       case lir_shl:  __ shll(value, count); break;
2984       case lir_shr:  __ sarl(value, count); break;
2985       case lir_ushr: __ shrl(value, count); break;
2986       default: ShouldNotReachHere();
2987     }
2988   } else if (dest->is_double_cpu()) {
2989 #ifndef _LP64
2990     Unimplemented();
2991 #else
2992     // first move left into dest so that left is not destroyed by the shift
2993     Register value = dest->as_register_lo();
2994     count = count & 0x1F; // Java spec
2995 
2996     move_regs(left->as_register_lo(), value);
2997     switch (code) {
2998       case lir_shl:  __ shlptr(value, count); break;
2999       case lir_shr:  __ sarptr(value, count); break;
3000       case lir_ushr: __ shrptr(value, count); break;
3001       default: ShouldNotReachHere();
3002     }
3003 #endif // _LP64
3004   } else {
3005     ShouldNotReachHere();
3006   }
3007 }
3008 
3009 
3010 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
3011   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3012   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3013   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3014   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
3015 }
3016 
3017 
3018 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
3019   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3020   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3021   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3022   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3023 }
3024 
3025 
3026 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
3027   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3028   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3029   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3030   __ movoop (Address(rsp, offset_from_rsp_in_bytes), o);
3031 }
3032 
3033 
3034 // This code replaces a call to arraycopy; no exception may
3035 // be thrown in this code, they must be thrown in the System.arraycopy
3036 // activation frame; we could save some checks if this would not be the case
3037 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3038   ciArrayKlass* default_type = op->expected_type();
3039   Register src = op->src()->as_register();
3040   Register dst = op->dst()->as_register();
3041   Register src_pos = op->src_pos()->as_register();
3042   Register dst_pos = op->dst_pos()->as_register();
3043   Register length  = op->length()->as_register();
3044   Register tmp = op->tmp()->as_register();
3045 
3046   CodeStub* stub = op->stub();
3047   int flags = op->flags();
3048   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
3049   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
3050 
3051   // if we don't know anything, just go through the generic arraycopy
3052   if (default_type == NULL) {
3053     Label done;
3054     // save outgoing arguments on stack in case call to System.arraycopy is needed
3055     // HACK ALERT. This code used to push the parameters in a hardwired fashion
3056     // for interpreter calling conventions. Now we have to do it in new style conventions.
3057     // For the moment until C1 gets the new register allocator I just force all the
3058     // args to the right place (except the register args) and then on the back side
3059     // reload the register args properly if we go slow path. Yuck
3060 
3061     // These are proper for the calling convention
3062     store_parameter(length, 2);
3063     store_parameter(dst_pos, 1);
3064     store_parameter(dst, 0);
3065 
3066     // these are just temporary placements until we need to reload
3067     store_parameter(src_pos, 3);
3068     store_parameter(src, 4);
3069     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3070 
3071     address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
3072 
3073     address copyfunc_addr = StubRoutines::generic_arraycopy();
3074 
3075     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
3076 #ifdef _LP64
3077     // The arguments are in java calling convention so we can trivially shift them to C
3078     // convention
3079     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
3080     __ mov(c_rarg0, j_rarg0);
3081     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
3082     __ mov(c_rarg1, j_rarg1);
3083     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
3084     __ mov(c_rarg2, j_rarg2);
3085     assert_different_registers(c_rarg3, j_rarg4);
3086     __ mov(c_rarg3, j_rarg3);
3087 #ifdef _WIN64
3088     // Allocate abi space for args but be sure to keep stack aligned
3089     __ subptr(rsp, 6*wordSize);
3090     store_parameter(j_rarg4, 4);
3091     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
3092       __ call(RuntimeAddress(C_entry));
3093     } else {
3094 #ifndef PRODUCT
3095       if (PrintC1Statistics) {
3096         __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3097       }
3098 #endif
3099       __ call(RuntimeAddress(copyfunc_addr));
3100     }
3101     __ addptr(rsp, 6*wordSize);
3102 #else
3103     __ mov(c_rarg4, j_rarg4);
3104     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
3105       __ call(RuntimeAddress(C_entry));
3106     } else {
3107 #ifndef PRODUCT
3108       if (PrintC1Statistics) {
3109         __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3110       }
3111 #endif
3112       __ call(RuntimeAddress(copyfunc_addr));
3113     }
3114 #endif // _WIN64
3115 #else
3116     __ push(length);
3117     __ push(dst_pos);
3118     __ push(dst);
3119     __ push(src_pos);
3120     __ push(src);
3121 
3122     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
3123       __ call_VM_leaf(C_entry, 5); // removes pushed parameter from the stack
3124     } else {
3125 #ifndef PRODUCT
3126       if (PrintC1Statistics) {
3127         __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3128       }
3129 #endif
3130       __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
3131     }
3132 
3133 #endif // _LP64
3134 
3135     __ cmpl(rax, 0);
3136     __ jcc(Assembler::equal, *stub->continuation());
3137 
3138     if (copyfunc_addr != NULL) {
3139       __ mov(tmp, rax);
3140       __ xorl(tmp, -1);
3141     }
3142 
3143     // Reload values from the stack so they are where the stub
3144     // expects them.
3145     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3146     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3147     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3148     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3149     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3150 
3151     if (copyfunc_addr != NULL) {
3152       __ subl(length, tmp);
3153       __ addl(src_pos, tmp);
3154       __ addl(dst_pos, tmp);
3155     }
3156     __ jmp(*stub->entry());
3157 
3158     __ bind(*stub->continuation());
3159     return;
3160   }
3161 
3162   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3163 
3164   int elem_size = type2aelembytes(basic_type);
3165   int shift_amount;
3166   Address::ScaleFactor scale;
3167 
3168   switch (elem_size) {
3169     case 1 :
3170       shift_amount = 0;
3171       scale = Address::times_1;
3172       break;
3173     case 2 :
3174       shift_amount = 1;
3175       scale = Address::times_2;
3176       break;
3177     case 4 :
3178       shift_amount = 2;
3179       scale = Address::times_4;
3180       break;
3181     case 8 :
3182       shift_amount = 3;
3183       scale = Address::times_8;
3184       break;
3185     default:
3186       ShouldNotReachHere();
3187   }
3188 
3189   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3190   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3191   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3192   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3193 
3194   // length and pos's are all sign extended at this point on 64bit
3195 
3196   // test for NULL
3197   if (flags & LIR_OpArrayCopy::src_null_check) {
3198     __ testptr(src, src);
3199     __ jcc(Assembler::zero, *stub->entry());
3200   }
3201   if (flags & LIR_OpArrayCopy::dst_null_check) {
3202     __ testptr(dst, dst);
3203     __ jcc(Assembler::zero, *stub->entry());
3204   }
3205 
3206   // check if negative
3207   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3208     __ testl(src_pos, src_pos);
3209     __ jcc(Assembler::less, *stub->entry());
3210   }
3211   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3212     __ testl(dst_pos, dst_pos);
3213     __ jcc(Assembler::less, *stub->entry());
3214   }
3215 
3216   if (flags & LIR_OpArrayCopy::src_range_check) {
3217     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3218     __ cmpl(tmp, src_length_addr);
3219     __ jcc(Assembler::above, *stub->entry());
3220   }
3221   if (flags & LIR_OpArrayCopy::dst_range_check) {
3222     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3223     __ cmpl(tmp, dst_length_addr);
3224     __ jcc(Assembler::above, *stub->entry());
3225   }
3226 
3227   if (flags & LIR_OpArrayCopy::length_positive_check) {
3228     __ testl(length, length);
3229     __ jcc(Assembler::less, *stub->entry());
3230     __ jcc(Assembler::zero, *stub->continuation());
3231   }
3232 
3233 #ifdef _LP64
3234   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3235   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3236 #endif
3237 
3238   if (flags & LIR_OpArrayCopy::type_check) {
3239     // We don't know the array types are compatible
3240     if (basic_type != T_OBJECT) {
3241       // Simple test for basic type arrays
3242       if (UseCompressedOops) {
3243         __ movl(tmp, src_klass_addr);
3244         __ cmpl(tmp, dst_klass_addr);
3245       } else {
3246         __ movptr(tmp, src_klass_addr);
3247         __ cmpptr(tmp, dst_klass_addr);
3248       }
3249       __ jcc(Assembler::notEqual, *stub->entry());
3250     } else {
3251       // For object arrays, if src is a sub class of dst then we can
3252       // safely do the copy.
3253       Label cont, slow;
3254 
3255       __ push(src);
3256       __ push(dst);
3257 
3258       __ load_klass(src, src);
3259       __ load_klass(dst, dst);
3260 
3261       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
3262 
3263       __ push(src);
3264       __ push(dst);
3265       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3266       __ pop(dst);
3267       __ pop(src);
3268 
3269       __ cmpl(src, 0);
3270       __ jcc(Assembler::notEqual, cont);
3271 
3272       __ bind(slow);
3273       __ pop(dst);
3274       __ pop(src);
3275 
3276       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
3277       if (copyfunc_addr != NULL) { // use stub if available
3278         // src is not a sub class of dst so we have to do a
3279         // per-element check.
3280 
3281         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
3282         if ((flags & mask) != mask) {
3283           // Check that at least both of them object arrays.
3284           assert(flags & mask, "one of the two should be known to be an object array");
3285 
3286           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3287             __ load_klass(tmp, src);
3288           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3289             __ load_klass(tmp, dst);
3290           }
3291           int lh_offset = in_bytes(Klass::layout_helper_offset());
3292           Address klass_lh_addr(tmp, lh_offset);
3293           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3294           __ cmpl(klass_lh_addr, objArray_lh);
3295           __ jcc(Assembler::notEqual, *stub->entry());
3296         }
3297 
3298        // Spill because stubs can use any register they like and it's
3299        // easier to restore just those that we care about.
3300        store_parameter(dst, 0);
3301        store_parameter(dst_pos, 1);
3302        store_parameter(length, 2);
3303        store_parameter(src_pos, 3);
3304        store_parameter(src, 4);
3305 
3306 #ifndef _LP64
3307         __ movptr(tmp, dst_klass_addr);
3308         __ movptr(tmp, Address(tmp, objArrayKlass::element_klass_offset()));
3309         __ push(tmp);
3310         __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3311         __ push(tmp);
3312         __ push(length);
3313         __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3314         __ push(tmp);
3315         __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3316         __ push(tmp);
3317 
3318         __ call_VM_leaf(copyfunc_addr, 5);
3319 #else
3320         __ movl2ptr(length, length); //higher 32bits must be null
3321 
3322         __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3323         assert_different_registers(c_rarg0, dst, dst_pos, length);
3324         __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3325         assert_different_registers(c_rarg1, dst, length);
3326 
3327         __ mov(c_rarg2, length);
3328         assert_different_registers(c_rarg2, dst);
3329 
3330 #ifdef _WIN64
3331         // Allocate abi space for args but be sure to keep stack aligned
3332         __ subptr(rsp, 6*wordSize);
3333         __ load_klass(c_rarg3, dst);
3334         __ movptr(c_rarg3, Address(c_rarg3, objArrayKlass::element_klass_offset()));
3335         store_parameter(c_rarg3, 4);
3336         __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
3337         __ call(RuntimeAddress(copyfunc_addr));
3338         __ addptr(rsp, 6*wordSize);
3339 #else
3340         __ load_klass(c_rarg4, dst);
3341         __ movptr(c_rarg4, Address(c_rarg4, objArrayKlass::element_klass_offset()));
3342         __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
3343         __ call(RuntimeAddress(copyfunc_addr));
3344 #endif
3345 
3346 #endif
3347 
3348 #ifndef PRODUCT
3349         if (PrintC1Statistics) {
3350           Label failed;
3351           __ testl(rax, rax);
3352           __ jcc(Assembler::notZero, failed);
3353           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
3354           __ bind(failed);
3355         }
3356 #endif
3357 
3358         __ testl(rax, rax);
3359         __ jcc(Assembler::zero, *stub->continuation());
3360 
3361 #ifndef PRODUCT
3362         if (PrintC1Statistics) {
3363           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
3364         }
3365 #endif
3366 
3367         __ mov(tmp, rax);
3368 
3369         __ xorl(tmp, -1);
3370 
3371         // Restore previously spilled arguments
3372         __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3373         __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3374         __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3375         __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3376         __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3377 
3378 
3379         __ subl(length, tmp);
3380         __ addl(src_pos, tmp);
3381         __ addl(dst_pos, tmp);
3382       }
3383 
3384       __ jmp(*stub->entry());
3385 
3386       __ bind(cont);
3387       __ pop(dst);
3388       __ pop(src);
3389     }
3390   }
3391 
3392 #ifdef ASSERT
3393   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3394     // Sanity check the known type with the incoming class.  For the
3395     // primitive case the types must match exactly with src.klass and
3396     // dst.klass each exactly matching the default type.  For the
3397     // object array case, if no type check is needed then either the
3398     // dst type is exactly the expected type and the src type is a
3399     // subtype which we can't check or src is the same array as dst
3400     // but not necessarily exactly of type default_type.
3401     Label known_ok, halt;
3402     __ movoop(tmp, default_type->constant_encoding());
3403 #ifdef _LP64
3404     if (UseCompressedOops) {
3405       __ encode_heap_oop(tmp);
3406     }
3407 #endif
3408 
3409     if (basic_type != T_OBJECT) {
3410 
3411       if (UseCompressedOops) __ cmpl(tmp, dst_klass_addr);
3412       else                   __ cmpptr(tmp, dst_klass_addr);
3413       __ jcc(Assembler::notEqual, halt);
3414       if (UseCompressedOops) __ cmpl(tmp, src_klass_addr);
3415       else                   __ cmpptr(tmp, src_klass_addr);
3416       __ jcc(Assembler::equal, known_ok);
3417     } else {
3418       if (UseCompressedOops) __ cmpl(tmp, dst_klass_addr);
3419       else                   __ cmpptr(tmp, dst_klass_addr);
3420       __ jcc(Assembler::equal, known_ok);
3421       __ cmpptr(src, dst);
3422       __ jcc(Assembler::equal, known_ok);
3423     }
3424     __ bind(halt);
3425     __ stop("incorrect type information in arraycopy");
3426     __ bind(known_ok);
3427   }
3428 #endif
3429 
3430 #ifndef PRODUCT
3431   if (PrintC1Statistics) {
3432     __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
3433   }
3434 #endif
3435 
3436 #ifdef _LP64
3437   assert_different_registers(c_rarg0, dst, dst_pos, length);
3438   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3439   assert_different_registers(c_rarg1, length);
3440   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3441   __ mov(c_rarg2, length);
3442 
3443 #else
3444   __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3445   store_parameter(tmp, 0);
3446   __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3447   store_parameter(tmp, 1);
3448   store_parameter(length, 2);
3449 #endif // _LP64
3450 
3451   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3452   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3453   const char *name;
3454   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3455   __ call_VM_leaf(entry, 0);
3456 
3457   __ bind(*stub->continuation());
3458 }
3459 
3460 
3461 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3462   Register obj = op->obj_opr()->as_register();  // may not be an oop
3463   Register hdr = op->hdr_opr()->as_register();
3464   Register lock = op->lock_opr()->as_register();
3465   if (!UseFastLocking) {
3466     __ jmp(*op->stub()->entry());
3467   } else if (op->code() == lir_lock) {
3468     Register scratch = noreg;
3469     if (UseBiasedLocking) {
3470       scratch = op->scratch_opr()->as_register();
3471     }
3472     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3473     // add debug info for NullPointerException only if one is possible
3474     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
3475     if (op->info() != NULL) {
3476       add_debug_info_for_null_check(null_check_offset, op->info());
3477     }
3478     // done
3479   } else if (op->code() == lir_unlock) {
3480     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3481     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3482   } else {
3483     Unimplemented();
3484   }
3485   __ bind(*op->stub()->continuation());
3486 }
3487 
3488 
3489 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3490   ciMethod* method = op->profiled_method();
3491   int bci          = op->profiled_bci();
3492 
3493   // Update counter for all call types
3494   ciMethodData* md = method->method_data_or_null();
3495   assert(md != NULL, "Sanity");
3496   ciProfileData* data = md->bci_to_data(bci);
3497   assert(data->is_CounterData(), "need CounterData for calls");
3498   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3499   Register mdo  = op->mdo()->as_register();
3500   __ movoop(mdo, md->constant_encoding());
3501   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3502   Bytecodes::Code bc = method->java_code_at_bci(bci);
3503   // Perform additional virtual call profiling for invokevirtual and
3504   // invokeinterface bytecodes
3505   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
3506       C1ProfileVirtualCalls) {
3507     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3508     Register recv = op->recv()->as_register();
3509     assert_different_registers(mdo, recv);
3510     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3511     ciKlass* known_klass = op->known_holder();
3512     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3513       // We know the type that will be seen at this call site; we can
3514       // statically update the methodDataOop rather than needing to do
3515       // dynamic tests on the receiver type
3516 
3517       // NOTE: we should probably put a lock around this search to
3518       // avoid collisions by concurrent compilations
3519       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3520       uint i;
3521       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3522         ciKlass* receiver = vc_data->receiver(i);
3523         if (known_klass->equals(receiver)) {
3524           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3525           __ addptr(data_addr, DataLayout::counter_increment);
3526           return;
3527         }
3528       }
3529 
3530       // Receiver type not found in profile data; select an empty slot
3531 
3532       // Note that this is less efficient than it should be because it
3533       // always does a write to the receiver part of the
3534       // VirtualCallData rather than just the first time
3535       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3536         ciKlass* receiver = vc_data->receiver(i);
3537         if (receiver == NULL) {
3538           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3539           __ movoop(recv_addr, known_klass->constant_encoding());
3540           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3541           __ addptr(data_addr, DataLayout::counter_increment);
3542           return;
3543         }
3544       }
3545     } else {
3546       __ load_klass(recv, recv);
3547       Label update_done;
3548       type_profile_helper(mdo, md, data, recv, &update_done);
3549       // Receiver did not match any saved receiver and there is no empty row for it.
3550       // Increment total counter to indicate polymorphic case.
3551       __ addptr(counter_addr, DataLayout::counter_increment);
3552 
3553       __ bind(update_done);
3554     }
3555   } else {
3556     // Static call
3557     __ addptr(counter_addr, DataLayout::counter_increment);
3558   }
3559 }
3560 
3561 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3562   Unimplemented();
3563 }
3564 
3565 
3566 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3567   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3568 }
3569 
3570 
3571 void LIR_Assembler::align_backward_branch_target() {
3572   __ align(BytesPerWord);
3573 }
3574 
3575 
3576 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
3577   if (left->is_single_cpu()) {
3578     __ negl(left->as_register());
3579     move_regs(left->as_register(), dest->as_register());
3580 
3581   } else if (left->is_double_cpu()) {
3582     Register lo = left->as_register_lo();
3583 #ifdef _LP64
3584     Register dst = dest->as_register_lo();
3585     __ movptr(dst, lo);
3586     __ negptr(dst);
3587 #else
3588     Register hi = left->as_register_hi();
3589     __ lneg(hi, lo);
3590     if (dest->as_register_lo() == hi) {
3591       assert(dest->as_register_hi() != lo, "destroying register");
3592       move_regs(hi, dest->as_register_hi());
3593       move_regs(lo, dest->as_register_lo());
3594     } else {
3595       move_regs(lo, dest->as_register_lo());
3596       move_regs(hi, dest->as_register_hi());
3597     }
3598 #endif // _LP64
3599 
3600   } else if (dest->is_single_xmm()) {
3601     if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3602       __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3603     }
3604     __ xorps(dest->as_xmm_float_reg(),
3605              ExternalAddress((address)float_signflip_pool));
3606 
3607   } else if (dest->is_double_xmm()) {
3608     if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3609       __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3610     }
3611     __ xorpd(dest->as_xmm_double_reg(),
3612              ExternalAddress((address)double_signflip_pool));
3613 
3614   } else if (left->is_single_fpu() || left->is_double_fpu()) {
3615     assert(left->fpu() == 0, "arg must be on TOS");
3616     assert(dest->fpu() == 0, "dest must be TOS");
3617     __ fchs();
3618 
3619   } else {
3620     ShouldNotReachHere();
3621   }
3622 }
3623 
3624 
3625 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
3626   assert(addr->is_address() && dest->is_register(), "check");
3627   Register reg;
3628   reg = dest->as_pointer_register();
3629   __ lea(reg, as_Address(addr->as_address_ptr()));
3630 }
3631 
3632 
3633 
3634 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3635   assert(!tmp->is_valid(), "don't need temporary");
3636   __ call(RuntimeAddress(dest));
3637   if (info != NULL) {
3638     add_call_info_here(info);
3639   }
3640 }
3641 
3642 
3643 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3644   assert(type == T_LONG, "only for volatile long fields");
3645 
3646   if (info != NULL) {
3647     add_debug_info_for_null_check_here(info);
3648   }
3649 
3650   if (src->is_double_xmm()) {
3651     if (dest->is_double_cpu()) {
3652 #ifdef _LP64
3653       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3654 #else
3655       __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
3656       __ psrlq(src->as_xmm_double_reg(), 32);
3657       __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
3658 #endif // _LP64
3659     } else if (dest->is_double_stack()) {
3660       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3661     } else if (dest->is_address()) {
3662       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3663     } else {
3664       ShouldNotReachHere();
3665     }
3666 
3667   } else if (dest->is_double_xmm()) {
3668     if (src->is_double_stack()) {
3669       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3670     } else if (src->is_address()) {
3671       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3672     } else {
3673       ShouldNotReachHere();
3674     }
3675 
3676   } else if (src->is_double_fpu()) {
3677     assert(src->fpu_regnrLo() == 0, "must be TOS");
3678     if (dest->is_double_stack()) {
3679       __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
3680     } else if (dest->is_address()) {
3681       __ fistp_d(as_Address(dest->as_address_ptr()));
3682     } else {
3683       ShouldNotReachHere();
3684     }
3685 
3686   } else if (dest->is_double_fpu()) {
3687     assert(dest->fpu_regnrLo() == 0, "must be TOS");
3688     if (src->is_double_stack()) {
3689       __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
3690     } else if (src->is_address()) {
3691       __ fild_d(as_Address(src->as_address_ptr()));
3692     } else {
3693       ShouldNotReachHere();
3694     }
3695   } else {
3696     ShouldNotReachHere();
3697   }
3698 }
3699 
3700 
3701 void LIR_Assembler::membar() {
3702   // QQQ sparc TSO uses this,
3703   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3704 }
3705 
3706 void LIR_Assembler::membar_acquire() {
3707   // No x86 machines currently require load fences
3708   // __ load_fence();
3709 }
3710 
3711 void LIR_Assembler::membar_release() {
3712   // No x86 machines currently require store fences
3713   // __ store_fence();
3714 }
3715 
3716 void LIR_Assembler::membar_loadload() {
3717   // no-op
3718   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3719 }
3720 
3721 void LIR_Assembler::membar_storestore() {
3722   // no-op
3723   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3724 }
3725 
3726 void LIR_Assembler::membar_loadstore() {
3727   // no-op
3728   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3729 }
3730 
3731 void LIR_Assembler::membar_storeload() {
3732   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3733 }
3734 
3735 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3736   assert(result_reg->is_register(), "check");
3737 #ifdef _LP64
3738   // __ get_thread(result_reg->as_register_lo());
3739   __ mov(result_reg->as_register(), r15_thread);
3740 #else
3741   __ get_thread(result_reg->as_register());
3742 #endif // _LP64
3743 }
3744 
3745 
3746 void LIR_Assembler::peephole(LIR_List*) {
3747   // do nothing for now
3748 }
3749 
3750 
3751 #undef __