1 /*
   2  * Copyright 2000-2010 Sun Microsystems, Inc.  All Rights Reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  20  * CA 95054 USA or visit www.sun.com if you need additional information or
  21  * have any questions.
  22  *
  23  */
  24 
  25 # include "incls/_precompiled.incl"
  26 # include "incls/_c1_LIRAssembler_x86.cpp.incl"
  27 
  28 
  29 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  30 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  31 // fast versions of NegF/NegD and AbsF/AbsD.
  32 
  33 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  34 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  35   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  36   // of 128-bits operands for SSE instructions.
  37   jlong *operand = (jlong*)(((long)adr)&((long)(~0xF)));
  38   // Store the value to a 128-bits operand.
  39   operand[0] = lo;
  40   operand[1] = hi;
  41   return operand;
  42 }
  43 
  44 // Buffer for 128-bits masks used by SSE instructions.
  45 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  46 
  47 // Static initialization during VM startup.
  48 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
  49 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
  50 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));
  51 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
  52 
  53 
  54 
  55 NEEDS_CLEANUP // remove this definitions ?
  56 const Register IC_Klass    = rax;   // where the IC klass is cached
  57 const Register SYNC_header = rax;   // synchronization header
  58 const Register SHIFT_count = rcx;   // where count for shift operations must be
  59 
  60 #define __ _masm->
  61 
  62 
  63 static void select_different_registers(Register preserve,
  64                                        Register extra,
  65                                        Register &tmp1,
  66                                        Register &tmp2) {
  67   if (tmp1 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp1 = extra;
  70   } else if (tmp2 == preserve) {
  71     assert_different_registers(tmp1, tmp2, extra);
  72     tmp2 = extra;
  73   }
  74   assert_different_registers(preserve, tmp1, tmp2);
  75 }
  76 
  77 
  78 
  79 static void select_different_registers(Register preserve,
  80                                        Register extra,
  81                                        Register &tmp1,
  82                                        Register &tmp2,
  83                                        Register &tmp3) {
  84   if (tmp1 == preserve) {
  85     assert_different_registers(tmp1, tmp2, tmp3, extra);
  86     tmp1 = extra;
  87   } else if (tmp2 == preserve) {
  88     assert_different_registers(tmp1, tmp2, tmp3, extra);
  89     tmp2 = extra;
  90   } else if (tmp3 == preserve) {
  91     assert_different_registers(tmp1, tmp2, tmp3, extra);
  92     tmp3 = extra;
  93   }
  94   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  95 }
  96 
  97 
  98 
  99 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 100   if (opr->is_constant()) {
 101     LIR_Const* constant = opr->as_constant_ptr();
 102     switch (constant->type()) {
 103       case T_INT: {
 104         return true;
 105       }
 106 
 107       default:
 108         return false;
 109     }
 110   }
 111   return false;
 112 }
 113 
 114 
 115 LIR_Opr LIR_Assembler::receiverOpr() {
 116   return FrameMap::receiver_opr;
 117 }
 118 
 119 LIR_Opr LIR_Assembler::incomingReceiverOpr() {
 120   return receiverOpr();
 121 }
 122 
 123 LIR_Opr LIR_Assembler::osrBufferPointer() {
 124   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 125 }
 126 
 127 //--------------fpu register translations-----------------------
 128 
 129 
 130 address LIR_Assembler::float_constant(float f) {
 131   address const_addr = __ float_constant(f);
 132   if (const_addr == NULL) {
 133     bailout("const section overflow");
 134     return __ code()->consts()->start();
 135   } else {
 136     return const_addr;
 137   }
 138 }
 139 
 140 
 141 address LIR_Assembler::double_constant(double d) {
 142   address const_addr = __ double_constant(d);
 143   if (const_addr == NULL) {
 144     bailout("const section overflow");
 145     return __ code()->consts()->start();
 146   } else {
 147     return const_addr;
 148   }
 149 }
 150 
 151 
 152 void LIR_Assembler::set_24bit_FPU() {
 153   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
 154 }
 155 
 156 void LIR_Assembler::reset_FPU() {
 157   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 158 }
 159 
 160 void LIR_Assembler::fpop() {
 161   __ fpop();
 162 }
 163 
 164 void LIR_Assembler::fxch(int i) {
 165   __ fxch(i);
 166 }
 167 
 168 void LIR_Assembler::fld(int i) {
 169   __ fld_s(i);
 170 }
 171 
 172 void LIR_Assembler::ffree(int i) {
 173   __ ffree(i);
 174 }
 175 
 176 void LIR_Assembler::breakpoint() {
 177   __ int3();
 178 }
 179 
 180 void LIR_Assembler::push(LIR_Opr opr) {
 181   if (opr->is_single_cpu()) {
 182     __ push_reg(opr->as_register());
 183   } else if (opr->is_double_cpu()) {
 184     NOT_LP64(__ push_reg(opr->as_register_hi()));
 185     __ push_reg(opr->as_register_lo());
 186   } else if (opr->is_stack()) {
 187     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 188   } else if (opr->is_constant()) {
 189     LIR_Const* const_opr = opr->as_constant_ptr();
 190     if (const_opr->type() == T_OBJECT) {
 191       __ push_oop(const_opr->as_jobject());
 192     } else if (const_opr->type() == T_INT) {
 193       __ push_jint(const_opr->as_jint());
 194     } else {
 195       ShouldNotReachHere();
 196     }
 197 
 198   } else {
 199     ShouldNotReachHere();
 200   }
 201 }
 202 
 203 void LIR_Assembler::pop(LIR_Opr opr) {
 204   if (opr->is_single_cpu()) {
 205     __ pop_reg(opr->as_register());
 206   } else {
 207     ShouldNotReachHere();
 208   }
 209 }
 210 
 211 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 212   return addr->base()->is_illegal() && addr->index()->is_illegal();
 213 }
 214 
 215 //-------------------------------------------
 216 
 217 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 218   return as_Address(addr, rscratch1);
 219 }
 220 
 221 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 222   if (addr->base()->is_illegal()) {
 223     assert(addr->index()->is_illegal(), "must be illegal too");
 224     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 225     if (! __ reachable(laddr)) {
 226       __ movptr(tmp, laddr.addr());
 227       Address res(tmp, 0);
 228       return res;
 229     } else {
 230       return __ as_Address(laddr);
 231     }
 232   }
 233 
 234   Register base = addr->base()->as_pointer_register();
 235 
 236   if (addr->index()->is_illegal()) {
 237     return Address( base, addr->disp());
 238   } else if (addr->index()->is_cpu_register()) {
 239     Register index = addr->index()->as_pointer_register();
 240     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 241   } else if (addr->index()->is_constant()) {
 242     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 243     assert(Assembler::is_simm32(addr_offset), "must be");
 244 
 245     return Address(base, addr_offset);
 246   } else {
 247     Unimplemented();
 248     return Address();
 249   }
 250 }
 251 
 252 
 253 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 254   Address base = as_Address(addr);
 255   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 256 }
 257 
 258 
 259 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 260   return as_Address(addr);
 261 }
 262 
 263 
 264 void LIR_Assembler::osr_entry() {
 265   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 266   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 267   ValueStack* entry_state = osr_entry->state();
 268   int number_of_locks = entry_state->locks_size();
 269 
 270   // we jump here if osr happens with the interpreter
 271   // state set up to continue at the beginning of the
 272   // loop that triggered osr - in particular, we have
 273   // the following registers setup:
 274   //
 275   // rcx: osr buffer
 276   //
 277 
 278   // build frame
 279   ciMethod* m = compilation()->method();
 280   __ build_frame(initial_frame_size_in_bytes());
 281 
 282   // OSR buffer is
 283   //
 284   // locals[nlocals-1..0]
 285   // monitors[0..number_of_locks]
 286   //
 287   // locals is a direct copy of the interpreter frame so in the osr buffer
 288   // so first slot in the local array is the last local from the interpreter
 289   // and last slot is local[0] (receiver) from the interpreter
 290   //
 291   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 292   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 293   // in the interpreter frame (the method lock if a sync method)
 294 
 295   // Initialize monitors in the compiled activation.
 296   //   rcx: pointer to osr buffer
 297   //
 298   // All other registers are dead at this point and the locals will be
 299   // copied into place by code emitted in the IR.
 300 
 301   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 302   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 303     int monitor_offset = BytesPerWord * method()->max_locals() +
 304       (2 * BytesPerWord) * (number_of_locks - 1);
 305     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 306     // the OSR buffer using 2 word entries: first the lock and then
 307     // the oop.
 308     for (int i = 0; i < number_of_locks; i++) {
 309       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 310 #ifdef ASSERT
 311       // verify the interpreter's monitor has a non-null object
 312       {
 313         Label L;
 314         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD);
 315         __ jcc(Assembler::notZero, L);
 316         __ stop("locked object is NULL");
 317         __ bind(L);
 318       }
 319 #endif
 320       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 321       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 322       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 323       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 324     }
 325   }
 326 }
 327 
 328 
 329 // inline cache check; done before the frame is built.
 330 int LIR_Assembler::check_icache() {
 331   Register receiver = FrameMap::receiver_opr->as_register();
 332   Register ic_klass = IC_Klass;
 333   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
 334 
 335   if (!VerifyOops) {
 336     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
 337     while ((__ offset() + ic_cmp_size) % CodeEntryAlignment != 0) {
 338       __ nop();
 339     }
 340   }
 341   int offset = __ offset();
 342   __ inline_cache_check(receiver, IC_Klass);
 343   assert(__ offset() % CodeEntryAlignment == 0 || VerifyOops, "alignment must be correct");
 344   if (VerifyOops) {
 345     // force alignment after the cache check.
 346     // It's been verified to be aligned if !VerifyOops
 347     __ align(CodeEntryAlignment);
 348   }
 349   return offset;
 350 }
 351 
 352 
 353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 354   jobject o = NULL;
 355   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 356   __ movoop(reg, o);
 357   patching_epilog(patch, lir_patch_normal, reg, info);
 358 }
 359 
 360 
 361 void LIR_Assembler::monitorexit(LIR_Opr obj_opr, LIR_Opr lock_opr, Register new_hdr, int monitor_no, Register exception) {
 362   if (exception->is_valid()) {
 363     // preserve exception
 364     // note: the monitor_exit runtime call is a leaf routine
 365     //       and cannot block => no GC can happen
 366     // The slow case (MonitorAccessStub) uses the first two stack slots
 367     // ([esp+0] and [esp+4]), therefore we store the exception at [esp+8]
 368     __ movptr (Address(rsp, 2*wordSize), exception);
 369   }
 370 
 371   Register obj_reg  = obj_opr->as_register();
 372   Register lock_reg = lock_opr->as_register();
 373 
 374   // setup registers (lock_reg must be rax, for lock_object)
 375   assert(obj_reg != SYNC_header && lock_reg != SYNC_header, "rax, must be available here");
 376   Register hdr = lock_reg;
 377   assert(new_hdr == SYNC_header, "wrong register");
 378   lock_reg = new_hdr;
 379   // compute pointer to BasicLock
 380   Address lock_addr = frame_map()->address_for_monitor_lock(monitor_no);
 381   __ lea(lock_reg, lock_addr);
 382   // unlock object
 383   MonitorAccessStub* slow_case = new MonitorExitStub(lock_opr, true, monitor_no);
 384   // _slow_case_stubs->append(slow_case);
 385   // temporary fix: must be created after exceptionhandler, therefore as call stub
 386   _slow_case_stubs->append(slow_case);
 387   if (UseFastLocking) {
 388     // try inlined fast unlocking first, revert to slow locking if it fails
 389     // note: lock_reg points to the displaced header since the displaced header offset is 0!
 390     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
 391     __ unlock_object(hdr, obj_reg, lock_reg, *slow_case->entry());
 392   } else {
 393     // always do slow unlocking
 394     // note: the slow unlocking code could be inlined here, however if we use
 395     //       slow unlocking, speed doesn't matter anyway and this solution is
 396     //       simpler and requires less duplicated code - additionally, the
 397     //       slow unlocking code is the same in either case which simplifies
 398     //       debugging
 399     __ jmp(*slow_case->entry());
 400   }
 401   // done
 402   __ bind(*slow_case->continuation());
 403 
 404   if (exception->is_valid()) {
 405     // restore exception
 406     __ movptr (exception, Address(rsp, 2 * wordSize));
 407   }
 408 }
 409 
 410 // This specifies the rsp decrement needed to build the frame
 411 int LIR_Assembler::initial_frame_size_in_bytes() {
 412   // if rounding, must let FrameMap know!
 413 
 414   // The frame_map records size in slots (32bit word)
 415 
 416   // subtract two words to account for return address and link
 417   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 418 }
 419 
 420 
 421 int LIR_Assembler::emit_exception_handler() {
 422   // if the last instruction is a call (typically to do a throw which
 423   // is coming at the end after block reordering) the return address
 424   // must still point into the code area in order to avoid assertion
 425   // failures when searching for the corresponding bci => add a nop
 426   // (was bug 5/14/1999 - gri)
 427   __ nop();
 428 
 429   // generate code for exception handler
 430   address handler_base = __ start_a_stub(exception_handler_size);
 431   if (handler_base == NULL) {
 432     // not enough space left for the handler
 433     bailout("exception handler overflow");
 434     return -1;
 435   }
 436 
 437   int offset = code_offset();
 438 
 439   // the exception oop and pc are in rax, and rdx
 440   // no other registers need to be preserved, so invalidate them
 441   __ invalidate_registers(false, true, true, false, true, true);
 442 
 443   // check that there is really an exception
 444   __ verify_not_null_oop(rax);
 445 
 446   // search an exception handler (rax: exception oop, rdx: throwing pc)
 447   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_nofpu_id)));
 448 
 449   __ stop("should not reach here");
 450 
 451   assert(code_offset() - offset <= exception_handler_size, "overflow");
 452   __ end_a_stub();
 453 
 454   return offset;
 455 }
 456 
 457 
 458 int LIR_Assembler::emit_deopt_handler() {
 459   // if the last instruction is a call (typically to do a throw which
 460   // is coming at the end after block reordering) the return address
 461   // must still point into the code area in order to avoid assertion
 462   // failures when searching for the corresponding bci => add a nop
 463   // (was bug 5/14/1999 - gri)
 464   __ nop();
 465 
 466   // generate code for exception handler
 467   address handler_base = __ start_a_stub(deopt_handler_size);
 468   if (handler_base == NULL) {
 469     // not enough space left for the handler
 470     bailout("deopt handler overflow");
 471     return -1;
 472   }
 473 
 474   int offset = code_offset();
 475   InternalAddress here(__ pc());
 476 
 477   __ pushptr(here.addr());
 478   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 479 
 480   assert(code_offset() - offset <= deopt_handler_size, "overflow");
 481   __ end_a_stub();
 482 
 483   return offset;
 484 }
 485 
 486 
 487 // This is the fast version of java.lang.String.compare; it has not
 488 // OSR-entry and therefore, we generate a slow version for OSR's
 489 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
 490   __ movptr (rbx, rcx); // receiver is in rcx
 491   __ movptr (rax, arg1->as_register());
 492 
 493   // Get addresses of first characters from both Strings
 494   __ movptr (rsi, Address(rax, java_lang_String::value_offset_in_bytes()));
 495   __ movptr (rcx, Address(rax, java_lang_String::offset_offset_in_bytes()));
 496   __ lea    (rsi, Address(rsi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 497 
 498 
 499   // rbx, may be NULL
 500   add_debug_info_for_null_check_here(info);
 501   __ movptr (rdi, Address(rbx, java_lang_String::value_offset_in_bytes()));
 502   __ movptr (rcx, Address(rbx, java_lang_String::offset_offset_in_bytes()));
 503   __ lea    (rdi, Address(rdi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 504 
 505   // compute minimum length (in rax) and difference of lengths (on top of stack)
 506   if (VM_Version::supports_cmov()) {
 507     __ movl     (rbx, Address(rbx, java_lang_String::count_offset_in_bytes()));
 508     __ movl     (rax, Address(rax, java_lang_String::count_offset_in_bytes()));
 509     __ mov      (rcx, rbx);
 510     __ subptr   (rbx, rax); // subtract lengths
 511     __ push     (rbx);      // result
 512     __ cmov     (Assembler::lessEqual, rax, rcx);
 513   } else {
 514     Label L;
 515     __ movl     (rbx, Address(rbx, java_lang_String::count_offset_in_bytes()));
 516     __ movl     (rcx, Address(rax, java_lang_String::count_offset_in_bytes()));
 517     __ mov      (rax, rbx);
 518     __ subptr   (rbx, rcx);
 519     __ push     (rbx);
 520     __ jcc      (Assembler::lessEqual, L);
 521     __ mov      (rax, rcx);
 522     __ bind (L);
 523   }
 524   // is minimum length 0?
 525   Label noLoop, haveResult;
 526   __ testptr (rax, rax);
 527   __ jcc (Assembler::zero, noLoop);
 528 
 529   // compare first characters
 530   __ load_unsigned_short(rcx, Address(rdi, 0));
 531   __ load_unsigned_short(rbx, Address(rsi, 0));
 532   __ subl(rcx, rbx);
 533   __ jcc(Assembler::notZero, haveResult);
 534   // starting loop
 535   __ decrement(rax); // we already tested index: skip one
 536   __ jcc(Assembler::zero, noLoop);
 537 
 538   // set rsi.edi to the end of the arrays (arrays have same length)
 539   // negate the index
 540 
 541   __ lea(rsi, Address(rsi, rax, Address::times_2, type2aelembytes(T_CHAR)));
 542   __ lea(rdi, Address(rdi, rax, Address::times_2, type2aelembytes(T_CHAR)));
 543   __ negptr(rax);
 544 
 545   // compare the strings in a loop
 546 
 547   Label loop;
 548   __ align(wordSize);
 549   __ bind(loop);
 550   __ load_unsigned_short(rcx, Address(rdi, rax, Address::times_2, 0));
 551   __ load_unsigned_short(rbx, Address(rsi, rax, Address::times_2, 0));
 552   __ subl(rcx, rbx);
 553   __ jcc(Assembler::notZero, haveResult);
 554   __ increment(rax);
 555   __ jcc(Assembler::notZero, loop);
 556 
 557   // strings are equal up to min length
 558 
 559   __ bind(noLoop);
 560   __ pop(rax);
 561   return_op(LIR_OprFact::illegalOpr);
 562 
 563   __ bind(haveResult);
 564   // leave instruction is going to discard the TOS value
 565   __ mov (rax, rcx); // result of call is in rax,
 566 }
 567 
 568 
 569 void LIR_Assembler::return_op(LIR_Opr result) {
 570   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 571   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 572     assert(result->fpu() == 0, "result must already be on TOS");
 573   }
 574 
 575   // Pop the stack before the safepoint code
 576   __ remove_frame(initial_frame_size_in_bytes());
 577 
 578   bool result_is_oop = result->is_valid() ? result->is_oop() : false;
 579 
 580   // Note: we do not need to round double result; float result has the right precision
 581   // the poll sets the condition code, but no data registers
 582   AddressLiteral polling_page(os::get_polling_page() + (SafepointPollOffset % os::vm_page_size()),
 583                               relocInfo::poll_return_type);
 584 
 585   // NOTE: the requires that the polling page be reachable else the reloc
 586   // goes to the movq that loads the address and not the faulting instruction
 587   // which breaks the signal handler code
 588 
 589   __ test32(rax, polling_page);
 590 
 591   __ ret(0);
 592 }
 593 
 594 
 595 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 596   AddressLiteral polling_page(os::get_polling_page() + (SafepointPollOffset % os::vm_page_size()),
 597                               relocInfo::poll_type);
 598 
 599   if (info != NULL) {
 600     add_debug_info_for_branch(info);
 601   } else {
 602     ShouldNotReachHere();
 603   }
 604 
 605   int offset = __ offset();
 606 
 607   // NOTE: the requires that the polling page be reachable else the reloc
 608   // goes to the movq that loads the address and not the faulting instruction
 609   // which breaks the signal handler code
 610 
 611   __ test32(rax, polling_page);
 612   return offset;
 613 }
 614 
 615 
 616 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 617   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 618 }
 619 
 620 void LIR_Assembler::swap_reg(Register a, Register b) {
 621   __ xchgptr(a, b);
 622 }
 623 
 624 
 625 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 626   assert(src->is_constant(), "should not call otherwise");
 627   assert(dest->is_register(), "should not call otherwise");
 628   LIR_Const* c = src->as_constant_ptr();
 629 
 630   switch (c->type()) {
 631     case T_INT:
 632     case T_ADDRESS: {
 633       assert(patch_code == lir_patch_none, "no patching handled here");
 634       __ movl(dest->as_register(), c->as_jint());
 635       break;
 636     }
 637 
 638     case T_LONG: {
 639       assert(patch_code == lir_patch_none, "no patching handled here");
 640 #ifdef _LP64
 641       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 642 #else
 643       __ movptr(dest->as_register_lo(), c->as_jint_lo());
 644       __ movptr(dest->as_register_hi(), c->as_jint_hi());
 645 #endif // _LP64
 646       break;
 647     }
 648 
 649     case T_OBJECT: {
 650       if (patch_code != lir_patch_none) {
 651         jobject2reg_with_patching(dest->as_register(), info);
 652       } else {
 653         __ movoop(dest->as_register(), c->as_jobject());
 654       }
 655       break;
 656     }
 657 
 658     case T_FLOAT: {
 659       if (dest->is_single_xmm()) {
 660         if (c->is_zero_float()) {
 661           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 662         } else {
 663           __ movflt(dest->as_xmm_float_reg(),
 664                    InternalAddress(float_constant(c->as_jfloat())));
 665         }
 666       } else {
 667         assert(dest->is_single_fpu(), "must be");
 668         assert(dest->fpu_regnr() == 0, "dest must be TOS");
 669         if (c->is_zero_float()) {
 670           __ fldz();
 671         } else if (c->is_one_float()) {
 672           __ fld1();
 673         } else {
 674           __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
 675         }
 676       }
 677       break;
 678     }
 679 
 680     case T_DOUBLE: {
 681       if (dest->is_double_xmm()) {
 682         if (c->is_zero_double()) {
 683           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 684         } else {
 685           __ movdbl(dest->as_xmm_double_reg(),
 686                     InternalAddress(double_constant(c->as_jdouble())));
 687         }
 688       } else {
 689         assert(dest->is_double_fpu(), "must be");
 690         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
 691         if (c->is_zero_double()) {
 692           __ fldz();
 693         } else if (c->is_one_double()) {
 694           __ fld1();
 695         } else {
 696           __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
 697         }
 698       }
 699       break;
 700     }
 701 
 702     default:
 703       ShouldNotReachHere();
 704   }
 705 }
 706 
 707 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 708   assert(src->is_constant(), "should not call otherwise");
 709   assert(dest->is_stack(), "should not call otherwise");
 710   LIR_Const* c = src->as_constant_ptr();
 711 
 712   switch (c->type()) {
 713     case T_INT:  // fall through
 714     case T_FLOAT:
 715     case T_ADDRESS:
 716       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 717       break;
 718 
 719     case T_OBJECT:
 720       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject());
 721       break;
 722 
 723     case T_LONG:  // fall through
 724     case T_DOUBLE:
 725 #ifdef _LP64
 726       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 727                                             lo_word_offset_in_bytes), (intptr_t)c->as_jlong_bits());
 728 #else
 729       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 730                                               lo_word_offset_in_bytes), c->as_jint_lo_bits());
 731       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 732                                               hi_word_offset_in_bytes), c->as_jint_hi_bits());
 733 #endif // _LP64
 734       break;
 735 
 736     default:
 737       ShouldNotReachHere();
 738   }
 739 }
 740 
 741 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info ) {
 742   assert(src->is_constant(), "should not call otherwise");
 743   assert(dest->is_address(), "should not call otherwise");
 744   LIR_Const* c = src->as_constant_ptr();
 745   LIR_Address* addr = dest->as_address_ptr();
 746 
 747   int null_check_here = code_offset();
 748   switch (type) {
 749     case T_INT:    // fall through
 750     case T_FLOAT:
 751     case T_ADDRESS:
 752       __ movl(as_Address(addr), c->as_jint_bits());
 753       break;
 754 
 755     case T_OBJECT:  // fall through
 756     case T_ARRAY:
 757       if (c->as_jobject() == NULL) {
 758         __ movptr(as_Address(addr), NULL_WORD);
 759       } else {
 760         if (is_literal_address(addr)) {
 761           ShouldNotReachHere();
 762           __ movoop(as_Address(addr, noreg), c->as_jobject());
 763         } else {
 764 #ifdef _LP64
 765           __ movoop(rscratch1, c->as_jobject());
 766           null_check_here = code_offset();
 767           __ movptr(as_Address_lo(addr), rscratch1);
 768 #else
 769           __ movoop(as_Address(addr), c->as_jobject());
 770 #endif
 771         }
 772       }
 773       break;
 774 
 775     case T_LONG:    // fall through
 776     case T_DOUBLE:
 777 #ifdef _LP64
 778       if (is_literal_address(addr)) {
 779         ShouldNotReachHere();
 780         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 781       } else {
 782         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 783         null_check_here = code_offset();
 784         __ movptr(as_Address_lo(addr), r10);
 785       }
 786 #else
 787       // Always reachable in 32bit so this doesn't produce useless move literal
 788       __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
 789       __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
 790 #endif // _LP64
 791       break;
 792 
 793     case T_BOOLEAN: // fall through
 794     case T_BYTE:
 795       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 796       break;
 797 
 798     case T_CHAR:    // fall through
 799     case T_SHORT:
 800       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 801       break;
 802 
 803     default:
 804       ShouldNotReachHere();
 805   };
 806 
 807   if (info != NULL) {
 808     add_debug_info_for_null_check(null_check_here, info);
 809   }
 810 }
 811 
 812 
 813 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 814   assert(src->is_register(), "should not call otherwise");
 815   assert(dest->is_register(), "should not call otherwise");
 816 
 817   // move between cpu-registers
 818   if (dest->is_single_cpu()) {
 819 #ifdef _LP64
 820     if (src->type() == T_LONG) {
 821       // Can do LONG -> OBJECT
 822       move_regs(src->as_register_lo(), dest->as_register());
 823       return;
 824     }
 825 #endif
 826     assert(src->is_single_cpu(), "must match");
 827     if (src->type() == T_OBJECT) {
 828       __ verify_oop(src->as_register());
 829     }
 830     move_regs(src->as_register(), dest->as_register());
 831 
 832   } else if (dest->is_double_cpu()) {
 833 #ifdef _LP64
 834     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 835       // Surprising to me but we can see move of a long to t_object
 836       __ verify_oop(src->as_register());
 837       move_regs(src->as_register(), dest->as_register_lo());
 838       return;
 839     }
 840 #endif
 841     assert(src->is_double_cpu(), "must match");
 842     Register f_lo = src->as_register_lo();
 843     Register f_hi = src->as_register_hi();
 844     Register t_lo = dest->as_register_lo();
 845     Register t_hi = dest->as_register_hi();
 846 #ifdef _LP64
 847     assert(f_hi == f_lo, "must be same");
 848     assert(t_hi == t_lo, "must be same");
 849     move_regs(f_lo, t_lo);
 850 #else
 851     assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
 852 
 853 
 854     if (f_lo == t_hi && f_hi == t_lo) {
 855       swap_reg(f_lo, f_hi);
 856     } else if (f_hi == t_lo) {
 857       assert(f_lo != t_hi, "overwriting register");
 858       move_regs(f_hi, t_hi);
 859       move_regs(f_lo, t_lo);
 860     } else {
 861       assert(f_hi != t_lo, "overwriting register");
 862       move_regs(f_lo, t_lo);
 863       move_regs(f_hi, t_hi);
 864     }
 865 #endif // LP64
 866 
 867     // special moves from fpu-register to xmm-register
 868     // necessary for method results
 869   } else if (src->is_single_xmm() && !dest->is_single_xmm()) {
 870     __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
 871     __ fld_s(Address(rsp, 0));
 872   } else if (src->is_double_xmm() && !dest->is_double_xmm()) {
 873     __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
 874     __ fld_d(Address(rsp, 0));
 875   } else if (dest->is_single_xmm() && !src->is_single_xmm()) {
 876     __ fstp_s(Address(rsp, 0));
 877     __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
 878   } else if (dest->is_double_xmm() && !src->is_double_xmm()) {
 879     __ fstp_d(Address(rsp, 0));
 880     __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
 881 
 882     // move between xmm-registers
 883   } else if (dest->is_single_xmm()) {
 884     assert(src->is_single_xmm(), "must match");
 885     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 886   } else if (dest->is_double_xmm()) {
 887     assert(src->is_double_xmm(), "must match");
 888     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 889 
 890     // move between fpu-registers (no instruction necessary because of fpu-stack)
 891   } else if (dest->is_single_fpu() || dest->is_double_fpu()) {
 892     assert(src->is_single_fpu() || src->is_double_fpu(), "must match");
 893     assert(src->fpu() == dest->fpu(), "currently should be nothing to do");
 894   } else {
 895     ShouldNotReachHere();
 896   }
 897 }
 898 
 899 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 900   assert(src->is_register(), "should not call otherwise");
 901   assert(dest->is_stack(), "should not call otherwise");
 902 
 903   if (src->is_single_cpu()) {
 904     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 905     if (type == T_OBJECT || type == T_ARRAY) {
 906       __ verify_oop(src->as_register());
 907       __ movptr (dst, src->as_register());
 908     } else {
 909       __ movl (dst, src->as_register());
 910     }
 911 
 912   } else if (src->is_double_cpu()) {
 913     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 914     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 915     __ movptr (dstLO, src->as_register_lo());
 916     NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
 917 
 918   } else if (src->is_single_xmm()) {
 919     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 920     __ movflt(dst_addr, src->as_xmm_float_reg());
 921 
 922   } else if (src->is_double_xmm()) {
 923     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 924     __ movdbl(dst_addr, src->as_xmm_double_reg());
 925 
 926   } else if (src->is_single_fpu()) {
 927     assert(src->fpu_regnr() == 0, "argument must be on TOS");
 928     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 929     if (pop_fpu_stack)     __ fstp_s (dst_addr);
 930     else                   __ fst_s  (dst_addr);
 931 
 932   } else if (src->is_double_fpu()) {
 933     assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
 934     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 935     if (pop_fpu_stack)     __ fstp_d (dst_addr);
 936     else                   __ fst_d  (dst_addr);
 937 
 938   } else {
 939     ShouldNotReachHere();
 940   }
 941 }
 942 
 943 
 944 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool /* unaligned */) {
 945   LIR_Address* to_addr = dest->as_address_ptr();
 946   PatchingStub* patch = NULL;
 947 
 948   if (type == T_ARRAY || type == T_OBJECT) {
 949     __ verify_oop(src->as_register());
 950   }
 951   if (patch_code != lir_patch_none) {
 952     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 953     Address toa = as_Address(to_addr);
 954     assert(toa.disp() != 0, "must have");
 955   }
 956   if (info != NULL) {
 957     add_debug_info_for_null_check_here(info);
 958   }
 959 
 960   switch (type) {
 961     case T_FLOAT: {
 962       if (src->is_single_xmm()) {
 963         __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
 964       } else {
 965         assert(src->is_single_fpu(), "must be");
 966         assert(src->fpu_regnr() == 0, "argument must be on TOS");
 967         if (pop_fpu_stack)      __ fstp_s(as_Address(to_addr));
 968         else                    __ fst_s (as_Address(to_addr));
 969       }
 970       break;
 971     }
 972 
 973     case T_DOUBLE: {
 974       if (src->is_double_xmm()) {
 975         __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
 976       } else {
 977         assert(src->is_double_fpu(), "must be");
 978         assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
 979         if (pop_fpu_stack)      __ fstp_d(as_Address(to_addr));
 980         else                    __ fst_d (as_Address(to_addr));
 981       }
 982       break;
 983     }
 984 
 985     case T_ADDRESS: // fall through
 986     case T_ARRAY:   // fall through
 987     case T_OBJECT:  // fall through
 988 #ifdef _LP64
 989       __ movptr(as_Address(to_addr), src->as_register());
 990       break;
 991 #endif // _LP64
 992     case T_INT:
 993       __ movl(as_Address(to_addr), src->as_register());
 994       break;
 995 
 996     case T_LONG: {
 997       Register from_lo = src->as_register_lo();
 998       Register from_hi = src->as_register_hi();
 999 #ifdef _LP64
1000       __ movptr(as_Address_lo(to_addr), from_lo);
1001 #else
1002       Register base = to_addr->base()->as_register();
1003       Register index = noreg;
1004       if (to_addr->index()->is_register()) {
1005         index = to_addr->index()->as_register();
1006       }
1007       if (base == from_lo || index == from_lo) {
1008         assert(base != from_hi, "can't be");
1009         assert(index == noreg || (index != base && index != from_hi), "can't handle this");
1010         __ movl(as_Address_hi(to_addr), from_hi);
1011         if (patch != NULL) {
1012           patching_epilog(patch, lir_patch_high, base, info);
1013           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1014           patch_code = lir_patch_low;
1015         }
1016         __ movl(as_Address_lo(to_addr), from_lo);
1017       } else {
1018         assert(index == noreg || (index != base && index != from_lo), "can't handle this");
1019         __ movl(as_Address_lo(to_addr), from_lo);
1020         if (patch != NULL) {
1021           patching_epilog(patch, lir_patch_low, base, info);
1022           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1023           patch_code = lir_patch_high;
1024         }
1025         __ movl(as_Address_hi(to_addr), from_hi);
1026       }
1027 #endif // _LP64
1028       break;
1029     }
1030 
1031     case T_BYTE:    // fall through
1032     case T_BOOLEAN: {
1033       Register src_reg = src->as_register();
1034       Address dst_addr = as_Address(to_addr);
1035       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
1036       __ movb(dst_addr, src_reg);
1037       break;
1038     }
1039 
1040     case T_CHAR:    // fall through
1041     case T_SHORT:
1042       __ movw(as_Address(to_addr), src->as_register());
1043       break;
1044 
1045     default:
1046       ShouldNotReachHere();
1047   }
1048 
1049   if (patch_code != lir_patch_none) {
1050     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1051   }
1052 }
1053 
1054 
1055 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1056   assert(src->is_stack(), "should not call otherwise");
1057   assert(dest->is_register(), "should not call otherwise");
1058 
1059   if (dest->is_single_cpu()) {
1060     if (type == T_ARRAY || type == T_OBJECT) {
1061       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1062       __ verify_oop(dest->as_register());
1063     } else {
1064       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1065     }
1066 
1067   } else if (dest->is_double_cpu()) {
1068     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1069     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1070     __ movptr(dest->as_register_lo(), src_addr_LO);
1071     NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1072 
1073   } else if (dest->is_single_xmm()) {
1074     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1075     __ movflt(dest->as_xmm_float_reg(), src_addr);
1076 
1077   } else if (dest->is_double_xmm()) {
1078     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1079     __ movdbl(dest->as_xmm_double_reg(), src_addr);
1080 
1081   } else if (dest->is_single_fpu()) {
1082     assert(dest->fpu_regnr() == 0, "dest must be TOS");
1083     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1084     __ fld_s(src_addr);
1085 
1086   } else if (dest->is_double_fpu()) {
1087     assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1088     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1089     __ fld_d(src_addr);
1090 
1091   } else {
1092     ShouldNotReachHere();
1093   }
1094 }
1095 
1096 
1097 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1098   if (src->is_single_stack()) {
1099     if (type == T_OBJECT || type == T_ARRAY) {
1100       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1101       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1102     } else {
1103 #ifndef _LP64
1104       __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1105       __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1106 #else
1107       //no pushl on 64bits
1108       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1109       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1110 #endif
1111     }
1112 
1113   } else if (src->is_double_stack()) {
1114 #ifdef _LP64
1115     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1116     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1117 #else
1118     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1119     // push and pop the part at src + wordSize, adding wordSize for the previous push
1120     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1121     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1122     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1123 #endif // _LP64
1124 
1125   } else {
1126     ShouldNotReachHere();
1127   }
1128 }
1129 
1130 
1131 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool /* unaligned */) {
1132   assert(src->is_address(), "should not call otherwise");
1133   assert(dest->is_register(), "should not call otherwise");
1134 
1135   LIR_Address* addr = src->as_address_ptr();
1136   Address from_addr = as_Address(addr);
1137 
1138   switch (type) {
1139     case T_BOOLEAN: // fall through
1140     case T_BYTE:    // fall through
1141     case T_CHAR:    // fall through
1142     case T_SHORT:
1143       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1144         // on pre P6 processors we may get partial register stalls
1145         // so blow away the value of to_rinfo before loading a
1146         // partial word into it.  Do it here so that it precedes
1147         // the potential patch point below.
1148         __ xorptr(dest->as_register(), dest->as_register());
1149       }
1150       break;
1151   }
1152 
1153   PatchingStub* patch = NULL;
1154   if (patch_code != lir_patch_none) {
1155     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1156     assert(from_addr.disp() != 0, "must have");
1157   }
1158   if (info != NULL) {
1159     add_debug_info_for_null_check_here(info);
1160   }
1161 
1162   switch (type) {
1163     case T_FLOAT: {
1164       if (dest->is_single_xmm()) {
1165         __ movflt(dest->as_xmm_float_reg(), from_addr);
1166       } else {
1167         assert(dest->is_single_fpu(), "must be");
1168         assert(dest->fpu_regnr() == 0, "dest must be TOS");
1169         __ fld_s(from_addr);
1170       }
1171       break;
1172     }
1173 
1174     case T_DOUBLE: {
1175       if (dest->is_double_xmm()) {
1176         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1177       } else {
1178         assert(dest->is_double_fpu(), "must be");
1179         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1180         __ fld_d(from_addr);
1181       }
1182       break;
1183     }
1184 
1185     case T_ADDRESS: // fall through
1186     case T_OBJECT:  // fall through
1187     case T_ARRAY:   // fall through
1188 #ifdef _LP64
1189       __ movptr(dest->as_register(), from_addr);
1190       break;
1191 #endif // _L64
1192     case T_INT:
1193       // %%% could this be a movl? this is safer but longer instruction
1194       __ movl2ptr(dest->as_register(), from_addr);
1195       break;
1196 
1197     case T_LONG: {
1198       Register to_lo = dest->as_register_lo();
1199       Register to_hi = dest->as_register_hi();
1200 #ifdef _LP64
1201       __ movptr(to_lo, as_Address_lo(addr));
1202 #else
1203       Register base = addr->base()->as_register();
1204       Register index = noreg;
1205       if (addr->index()->is_register()) {
1206         index = addr->index()->as_register();
1207       }
1208       if ((base == to_lo && index == to_hi) ||
1209           (base == to_hi && index == to_lo)) {
1210         // addresses with 2 registers are only formed as a result of
1211         // array access so this code will never have to deal with
1212         // patches or null checks.
1213         assert(info == NULL && patch == NULL, "must be");
1214         __ lea(to_hi, as_Address(addr));
1215         __ movl(to_lo, Address(to_hi, 0));
1216         __ movl(to_hi, Address(to_hi, BytesPerWord));
1217       } else if (base == to_lo || index == to_lo) {
1218         assert(base != to_hi, "can't be");
1219         assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1220         __ movl(to_hi, as_Address_hi(addr));
1221         if (patch != NULL) {
1222           patching_epilog(patch, lir_patch_high, base, info);
1223           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1224           patch_code = lir_patch_low;
1225         }
1226         __ movl(to_lo, as_Address_lo(addr));
1227       } else {
1228         assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1229         __ movl(to_lo, as_Address_lo(addr));
1230         if (patch != NULL) {
1231           patching_epilog(patch, lir_patch_low, base, info);
1232           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1233           patch_code = lir_patch_high;
1234         }
1235         __ movl(to_hi, as_Address_hi(addr));
1236       }
1237 #endif // _LP64
1238       break;
1239     }
1240 
1241     case T_BOOLEAN: // fall through
1242     case T_BYTE: {
1243       Register dest_reg = dest->as_register();
1244       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1245       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1246         __ movsbl(dest_reg, from_addr);
1247       } else {
1248         __ movb(dest_reg, from_addr);
1249         __ shll(dest_reg, 24);
1250         __ sarl(dest_reg, 24);
1251       }
1252       // These are unsigned so the zero extension on 64bit is just what we need
1253       break;
1254     }
1255 
1256     case T_CHAR: {
1257       Register dest_reg = dest->as_register();
1258       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1259       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1260         __ movzwl(dest_reg, from_addr);
1261       } else {
1262         __ movw(dest_reg, from_addr);
1263       }
1264       // This is unsigned so the zero extension on 64bit is just what we need
1265       // __ movl2ptr(dest_reg, dest_reg);
1266       break;
1267     }
1268 
1269     case T_SHORT: {
1270       Register dest_reg = dest->as_register();
1271       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1272         __ movswl(dest_reg, from_addr);
1273       } else {
1274         __ movw(dest_reg, from_addr);
1275         __ shll(dest_reg, 16);
1276         __ sarl(dest_reg, 16);
1277       }
1278       // Might not be needed in 64bit but certainly doesn't hurt (except for code size)
1279       __ movl2ptr(dest_reg, dest_reg);
1280       break;
1281     }
1282 
1283     default:
1284       ShouldNotReachHere();
1285   }
1286 
1287   if (patch != NULL) {
1288     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1289   }
1290 
1291   if (type == T_ARRAY || type == T_OBJECT) {
1292     __ verify_oop(dest->as_register());
1293   }
1294 }
1295 
1296 
1297 void LIR_Assembler::prefetchr(LIR_Opr src) {
1298   LIR_Address* addr = src->as_address_ptr();
1299   Address from_addr = as_Address(addr);
1300 
1301   if (VM_Version::supports_sse()) {
1302     switch (ReadPrefetchInstr) {
1303       case 0:
1304         __ prefetchnta(from_addr); break;
1305       case 1:
1306         __ prefetcht0(from_addr); break;
1307       case 2:
1308         __ prefetcht2(from_addr); break;
1309       default:
1310         ShouldNotReachHere(); break;
1311     }
1312   } else if (VM_Version::supports_3dnow()) {
1313     __ prefetchr(from_addr);
1314   }
1315 }
1316 
1317 
1318 void LIR_Assembler::prefetchw(LIR_Opr src) {
1319   LIR_Address* addr = src->as_address_ptr();
1320   Address from_addr = as_Address(addr);
1321 
1322   if (VM_Version::supports_sse()) {
1323     switch (AllocatePrefetchInstr) {
1324       case 0:
1325         __ prefetchnta(from_addr); break;
1326       case 1:
1327         __ prefetcht0(from_addr); break;
1328       case 2:
1329         __ prefetcht2(from_addr); break;
1330       case 3:
1331         __ prefetchw(from_addr); break;
1332       default:
1333         ShouldNotReachHere(); break;
1334     }
1335   } else if (VM_Version::supports_3dnow()) {
1336     __ prefetchw(from_addr);
1337   }
1338 }
1339 
1340 
1341 NEEDS_CLEANUP; // This could be static?
1342 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1343   int elem_size = type2aelembytes(type);
1344   switch (elem_size) {
1345     case 1: return Address::times_1;
1346     case 2: return Address::times_2;
1347     case 4: return Address::times_4;
1348     case 8: return Address::times_8;
1349   }
1350   ShouldNotReachHere();
1351   return Address::no_scale;
1352 }
1353 
1354 
1355 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1356   switch (op->code()) {
1357     case lir_idiv:
1358     case lir_irem:
1359       arithmetic_idiv(op->code(),
1360                       op->in_opr1(),
1361                       op->in_opr2(),
1362                       op->in_opr3(),
1363                       op->result_opr(),
1364                       op->info());
1365       break;
1366     default:      ShouldNotReachHere(); break;
1367   }
1368 }
1369 
1370 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1371 #ifdef ASSERT
1372   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1373   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1374   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1375 #endif
1376 
1377   if (op->cond() == lir_cond_always) {
1378     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1379     __ jmp (*(op->label()));
1380   } else {
1381     Assembler::Condition acond = Assembler::zero;
1382     if (op->code() == lir_cond_float_branch) {
1383       assert(op->ublock() != NULL, "must have unordered successor");
1384       __ jcc(Assembler::parity, *(op->ublock()->label()));
1385       switch(op->cond()) {
1386         case lir_cond_equal:        acond = Assembler::equal;      break;
1387         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1388         case lir_cond_less:         acond = Assembler::below;      break;
1389         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1390         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1391         case lir_cond_greater:      acond = Assembler::above;      break;
1392         default:                         ShouldNotReachHere();
1393       }
1394     } else {
1395       switch (op->cond()) {
1396         case lir_cond_equal:        acond = Assembler::equal;       break;
1397         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1398         case lir_cond_less:         acond = Assembler::less;        break;
1399         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1400         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1401         case lir_cond_greater:      acond = Assembler::greater;     break;
1402         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1403         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1404         default:                         ShouldNotReachHere();
1405       }
1406     }
1407     __ jcc(acond,*(op->label()));
1408   }
1409 }
1410 
1411 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1412   LIR_Opr src  = op->in_opr();
1413   LIR_Opr dest = op->result_opr();
1414 
1415   switch (op->bytecode()) {
1416     case Bytecodes::_i2l:
1417 #ifdef _LP64
1418       __ movl2ptr(dest->as_register_lo(), src->as_register());
1419 #else
1420       move_regs(src->as_register(), dest->as_register_lo());
1421       move_regs(src->as_register(), dest->as_register_hi());
1422       __ sarl(dest->as_register_hi(), 31);
1423 #endif // LP64
1424       break;
1425 
1426     case Bytecodes::_l2i:
1427       move_regs(src->as_register_lo(), dest->as_register());
1428       break;
1429 
1430     case Bytecodes::_i2b:
1431       move_regs(src->as_register(), dest->as_register());
1432       __ sign_extend_byte(dest->as_register());
1433       break;
1434 
1435     case Bytecodes::_i2c:
1436       move_regs(src->as_register(), dest->as_register());
1437       __ andl(dest->as_register(), 0xFFFF);
1438       break;
1439 
1440     case Bytecodes::_i2s:
1441       move_regs(src->as_register(), dest->as_register());
1442       __ sign_extend_short(dest->as_register());
1443       break;
1444 
1445 
1446     case Bytecodes::_f2d:
1447     case Bytecodes::_d2f:
1448       if (dest->is_single_xmm()) {
1449         __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1450       } else if (dest->is_double_xmm()) {
1451         __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1452       } else {
1453         assert(src->fpu() == dest->fpu(), "register must be equal");
1454         // do nothing (float result is rounded later through spilling)
1455       }
1456       break;
1457 
1458     case Bytecodes::_i2f:
1459     case Bytecodes::_i2d:
1460       if (dest->is_single_xmm()) {
1461         __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1462       } else if (dest->is_double_xmm()) {
1463         __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1464       } else {
1465         assert(dest->fpu() == 0, "result must be on TOS");
1466         __ movl(Address(rsp, 0), src->as_register());
1467         __ fild_s(Address(rsp, 0));
1468       }
1469       break;
1470 
1471     case Bytecodes::_f2i:
1472     case Bytecodes::_d2i:
1473       if (src->is_single_xmm()) {
1474         __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1475       } else if (src->is_double_xmm()) {
1476         __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1477       } else {
1478         assert(src->fpu() == 0, "input must be on TOS");
1479         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
1480         __ fist_s(Address(rsp, 0));
1481         __ movl(dest->as_register(), Address(rsp, 0));
1482         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1483       }
1484 
1485       // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
1486       assert(op->stub() != NULL, "stub required");
1487       __ cmpl(dest->as_register(), 0x80000000);
1488       __ jcc(Assembler::equal, *op->stub()->entry());
1489       __ bind(*op->stub()->continuation());
1490       break;
1491 
1492     case Bytecodes::_l2f:
1493     case Bytecodes::_l2d:
1494       assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)");
1495       assert(dest->fpu() == 0, "result must be on TOS");
1496 
1497       __ movptr(Address(rsp, 0),            src->as_register_lo());
1498       NOT_LP64(__ movl(Address(rsp, BytesPerWord), src->as_register_hi()));
1499       __ fild_d(Address(rsp, 0));
1500       // float result is rounded later through spilling
1501       break;
1502 
1503     case Bytecodes::_f2l:
1504     case Bytecodes::_d2l:
1505       assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)");
1506       assert(src->fpu() == 0, "input must be on TOS");
1507       assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers");
1508 
1509       // instruction sequence too long to inline it here
1510       {
1511         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1512       }
1513       break;
1514 
1515     default: ShouldNotReachHere();
1516   }
1517 }
1518 
1519 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1520   if (op->init_check()) {
1521     __ cmpl(Address(op->klass()->as_register(),
1522                     instanceKlass::init_state_offset_in_bytes() + sizeof(oopDesc)),
1523             instanceKlass::fully_initialized);
1524     add_debug_info_for_null_check_here(op->stub()->info());
1525     __ jcc(Assembler::notEqual, *op->stub()->entry());
1526   }
1527   __ allocate_object(op->obj()->as_register(),
1528                      op->tmp1()->as_register(),
1529                      op->tmp2()->as_register(),
1530                      op->header_size(),
1531                      op->object_size(),
1532                      op->klass()->as_register(),
1533                      *op->stub()->entry());
1534   __ bind(*op->stub()->continuation());
1535 }
1536 
1537 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1538   if (UseSlowPath ||
1539       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1540       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1541     __ jmp(*op->stub()->entry());
1542   } else {
1543     Register len =  op->len()->as_register();
1544     Register tmp1 = op->tmp1()->as_register();
1545     Register tmp2 = op->tmp2()->as_register();
1546     Register tmp3 = op->tmp3()->as_register();
1547     if (len == tmp1) {
1548       tmp1 = tmp3;
1549     } else if (len == tmp2) {
1550       tmp2 = tmp3;
1551     } else if (len == tmp3) {
1552       // everything is ok
1553     } else {
1554       __ mov(tmp3, len);
1555     }
1556     __ allocate_array(op->obj()->as_register(),
1557                       len,
1558                       tmp1,
1559                       tmp2,
1560                       arrayOopDesc::header_size(op->type()),
1561                       array_element_size(op->type()),
1562                       op->klass()->as_register(),
1563                       *op->stub()->entry());
1564   }
1565   __ bind(*op->stub()->continuation());
1566 }
1567 
1568 
1569 
1570 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1571   LIR_Code code = op->code();
1572   if (code == lir_store_check) {
1573     Register value = op->object()->as_register();
1574     Register array = op->array()->as_register();
1575     Register k_RInfo = op->tmp1()->as_register();
1576     Register klass_RInfo = op->tmp2()->as_register();
1577     Register Rtmp1 = op->tmp3()->as_register();
1578 
1579     CodeStub* stub = op->stub();
1580     Label done;
1581     __ cmpptr(value, (int32_t)NULL_WORD);
1582     __ jcc(Assembler::equal, done);
1583     add_debug_info_for_null_check_here(op->info_for_exception());
1584     __ movptr(k_RInfo, Address(array, oopDesc::klass_offset_in_bytes()));
1585     __ movptr(klass_RInfo, Address(value, oopDesc::klass_offset_in_bytes()));
1586 
1587     // get instance klass
1588     __ movptr(k_RInfo, Address(k_RInfo, objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc)));
1589     // perform the fast part of the checking logic
1590     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, &done, stub->entry(), NULL);
1591     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1592     __ push(klass_RInfo);
1593     __ push(k_RInfo);
1594     __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1595     __ pop(klass_RInfo);
1596     __ pop(k_RInfo);
1597     // result is a boolean
1598     __ cmpl(k_RInfo, 0);
1599     __ jcc(Assembler::equal, *stub->entry());
1600     __ bind(done);
1601   } else if (op->code() == lir_checkcast) {
1602     // we always need a stub for the failure case.
1603     CodeStub* stub = op->stub();
1604     Register obj = op->object()->as_register();
1605     Register k_RInfo = op->tmp1()->as_register();
1606     Register klass_RInfo = op->tmp2()->as_register();
1607     Register dst = op->result_opr()->as_register();
1608     ciKlass* k = op->klass();
1609     Register Rtmp1 = noreg;
1610 
1611     Label done;
1612     if (obj == k_RInfo) {
1613       k_RInfo = dst;
1614     } else if (obj == klass_RInfo) {
1615       klass_RInfo = dst;
1616     }
1617     if (k->is_loaded()) {
1618       select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1619     } else {
1620       Rtmp1 = op->tmp3()->as_register();
1621       select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1622     }
1623 
1624     assert_different_registers(obj, k_RInfo, klass_RInfo);
1625     if (!k->is_loaded()) {
1626       jobject2reg_with_patching(k_RInfo, op->info_for_patch());
1627     } else {
1628 #ifdef _LP64
1629       __ movoop(k_RInfo, k->constant_encoding());
1630 #else
1631       k_RInfo = noreg;
1632 #endif // _LP64
1633     }
1634     assert(obj != k_RInfo, "must be different");
1635     __ cmpptr(obj, (int32_t)NULL_WORD);
1636     if (op->profiled_method() != NULL) {
1637       ciMethod* method = op->profiled_method();
1638       int bci          = op->profiled_bci();
1639 
1640       Label profile_done;
1641       __ jcc(Assembler::notEqual, profile_done);
1642       // Object is null; update methodDataOop
1643       ciMethodData* md = method->method_data();
1644       if (md == NULL) {
1645         bailout("out of memory building methodDataOop");
1646         return;
1647       }
1648       ciProfileData* data = md->bci_to_data(bci);
1649       assert(data != NULL,       "need data for checkcast");
1650       assert(data->is_BitData(), "need BitData for checkcast");
1651       Register mdo  = klass_RInfo;
1652       __ movoop(mdo, md->constant_encoding());
1653       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
1654       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1655       __ orl(data_addr, header_bits);
1656       __ jmp(done);
1657       __ bind(profile_done);
1658     } else {
1659       __ jcc(Assembler::equal, done);
1660     }
1661     __ verify_oop(obj);
1662 
1663     if (op->fast_check()) {
1664       // get object classo
1665       // not a safepoint as obj null check happens earlier
1666       if (k->is_loaded()) {
1667 #ifdef _LP64
1668         __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1669 #else
1670         __ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1671 #endif // _LP64
1672       } else {
1673         __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1674 
1675       }
1676       __ jcc(Assembler::notEqual, *stub->entry());
1677       __ bind(done);
1678     } else {
1679       // get object class
1680       // not a safepoint as obj null check happens earlier
1681       __ movptr(klass_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1682       if (k->is_loaded()) {
1683         // See if we get an immediate positive hit
1684 #ifdef _LP64
1685         __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1686 #else
1687         __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1688 #endif // _LP64
1689         if (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() != k->super_check_offset()) {
1690           __ jcc(Assembler::notEqual, *stub->entry());
1691         } else {
1692           // See if we get an immediate positive hit
1693           __ jcc(Assembler::equal, done);
1694           // check for self
1695 #ifdef _LP64
1696           __ cmpptr(klass_RInfo, k_RInfo);
1697 #else
1698           __ cmpoop(klass_RInfo, k->constant_encoding());
1699 #endif // _LP64
1700           __ jcc(Assembler::equal, done);
1701 
1702           __ push(klass_RInfo);
1703 #ifdef _LP64
1704           __ push(k_RInfo);
1705 #else
1706           __ pushoop(k->constant_encoding());
1707 #endif // _LP64
1708           __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1709           __ pop(klass_RInfo);
1710           __ pop(klass_RInfo);
1711           // result is a boolean
1712           __ cmpl(klass_RInfo, 0);
1713           __ jcc(Assembler::equal, *stub->entry());
1714         }
1715         __ bind(done);
1716       } else {
1717         // perform the fast part of the checking logic
1718         __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, &done, stub->entry(), NULL);
1719         // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1720         __ push(klass_RInfo);
1721         __ push(k_RInfo);
1722         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1723         __ pop(klass_RInfo);
1724         __ pop(k_RInfo);
1725         // result is a boolean
1726         __ cmpl(k_RInfo, 0);
1727         __ jcc(Assembler::equal, *stub->entry());
1728         __ bind(done);
1729       }
1730 
1731     }
1732     if (dst != obj) {
1733       __ mov(dst, obj);
1734     }
1735   } else if (code == lir_instanceof) {
1736     Register obj = op->object()->as_register();
1737     Register k_RInfo = op->tmp1()->as_register();
1738     Register klass_RInfo = op->tmp2()->as_register();
1739     Register dst = op->result_opr()->as_register();
1740     ciKlass* k = op->klass();
1741 
1742     Label done;
1743     Label zero;
1744     Label one;
1745     if (obj == k_RInfo) {
1746       k_RInfo = klass_RInfo;
1747       klass_RInfo = obj;
1748     }
1749     // patching may screw with our temporaries on sparc,
1750     // so let's do it before loading the class
1751     if (!k->is_loaded()) {
1752       jobject2reg_with_patching(k_RInfo, op->info_for_patch());
1753     } else {
1754       LP64_ONLY(__ movoop(k_RInfo, k->constant_encoding()));
1755     }
1756     assert(obj != k_RInfo, "must be different");
1757 
1758     __ verify_oop(obj);
1759     if (op->fast_check()) {
1760       __ cmpptr(obj, (int32_t)NULL_WORD);
1761       __ jcc(Assembler::equal, zero);
1762       // get object class
1763       // not a safepoint as obj null check happens earlier
1764       if (LP64_ONLY(false &&) k->is_loaded()) {
1765         NOT_LP64(__ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding()));
1766         k_RInfo = noreg;
1767       } else {
1768         __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1769 
1770       }
1771       __ jcc(Assembler::equal, one);
1772     } else {
1773       // get object class
1774       // not a safepoint as obj null check happens earlier
1775       __ cmpptr(obj, (int32_t)NULL_WORD);
1776       __ jcc(Assembler::equal, zero);
1777       __ movptr(klass_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1778 
1779 #ifndef _LP64
1780       if (k->is_loaded()) {
1781         // See if we get an immediate positive hit
1782         __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1783         __ jcc(Assembler::equal, one);
1784         if (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() == k->super_check_offset()) {
1785           // check for self
1786           __ cmpoop(klass_RInfo, k->constant_encoding());
1787           __ jcc(Assembler::equal, one);
1788           __ push(klass_RInfo);
1789           __ pushoop(k->constant_encoding());
1790           __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1791           __ pop(klass_RInfo);
1792           __ pop(dst);
1793           __ jmp(done);
1794         }
1795       }
1796         else // next block is unconditional if LP64:
1797 #endif // LP64
1798       {
1799         assert(dst != klass_RInfo && dst != k_RInfo, "need 3 registers");
1800 
1801         // perform the fast part of the checking logic
1802         __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, dst, &one, &zero, NULL);
1803         // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1804         __ push(klass_RInfo);
1805         __ push(k_RInfo);
1806         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1807         __ pop(klass_RInfo);
1808         __ pop(dst);
1809         __ jmp(done);
1810       }
1811     }
1812     __ bind(zero);
1813     __ xorptr(dst, dst);
1814     __ jmp(done);
1815     __ bind(one);
1816     __ movptr(dst, 1);
1817     __ bind(done);
1818   } else {
1819     ShouldNotReachHere();
1820   }
1821 
1822 }
1823 
1824 
1825 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1826   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
1827     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1828     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1829     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1830     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1831     Register addr = op->addr()->as_register();
1832     if (os::is_MP()) {
1833       __ lock();
1834     }
1835     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1836 
1837   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1838     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1839     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1840     Register newval = op->new_value()->as_register();
1841     Register cmpval = op->cmp_value()->as_register();
1842     assert(cmpval == rax, "wrong register");
1843     assert(newval != NULL, "new val must be register");
1844     assert(cmpval != newval, "cmp and new values must be in different registers");
1845     assert(cmpval != addr, "cmp and addr must be in different registers");
1846     assert(newval != addr, "new value and addr must be in different registers");
1847     if (os::is_MP()) {
1848       __ lock();
1849     }
1850     if ( op->code() == lir_cas_obj) {
1851       __ cmpxchgptr(newval, Address(addr, 0));
1852     } else if (op->code() == lir_cas_int) {
1853       __ cmpxchgl(newval, Address(addr, 0));
1854     } else {
1855       LP64_ONLY(__ cmpxchgq(newval, Address(addr, 0)));
1856     }
1857 #ifdef _LP64
1858   } else if (op->code() == lir_cas_long) {
1859     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1860     Register newval = op->new_value()->as_register_lo();
1861     Register cmpval = op->cmp_value()->as_register_lo();
1862     assert(cmpval == rax, "wrong register");
1863     assert(newval != NULL, "new val must be register");
1864     assert(cmpval != newval, "cmp and new values must be in different registers");
1865     assert(cmpval != addr, "cmp and addr must be in different registers");
1866     assert(newval != addr, "new value and addr must be in different registers");
1867     if (os::is_MP()) {
1868       __ lock();
1869     }
1870     __ cmpxchgq(newval, Address(addr, 0));
1871 #endif // _LP64
1872   } else {
1873     Unimplemented();
1874   }
1875 }
1876 
1877 
1878 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result) {
1879   Assembler::Condition acond, ncond;
1880   switch (condition) {
1881     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1882     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1883     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1884     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1885     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1886     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1887     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1888     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1889     default:                    ShouldNotReachHere();
1890   }
1891 
1892   if (opr1->is_cpu_register()) {
1893     reg2reg(opr1, result);
1894   } else if (opr1->is_stack()) {
1895     stack2reg(opr1, result, result->type());
1896   } else if (opr1->is_constant()) {
1897     const2reg(opr1, result, lir_patch_none, NULL);
1898   } else {
1899     ShouldNotReachHere();
1900   }
1901 
1902   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1903     // optimized version that does not require a branch
1904     if (opr2->is_single_cpu()) {
1905       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1906       __ cmov(ncond, result->as_register(), opr2->as_register());
1907     } else if (opr2->is_double_cpu()) {
1908       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1909       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1910       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1911       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
1912     } else if (opr2->is_single_stack()) {
1913       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1914     } else if (opr2->is_double_stack()) {
1915       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1916       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
1917     } else {
1918       ShouldNotReachHere();
1919     }
1920 
1921   } else {
1922     Label skip;
1923     __ jcc (acond, skip);
1924     if (opr2->is_cpu_register()) {
1925       reg2reg(opr2, result);
1926     } else if (opr2->is_stack()) {
1927       stack2reg(opr2, result, result->type());
1928     } else if (opr2->is_constant()) {
1929       const2reg(opr2, result, lir_patch_none, NULL);
1930     } else {
1931       ShouldNotReachHere();
1932     }
1933     __ bind(skip);
1934   }
1935 }
1936 
1937 
1938 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1939   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1940 
1941   if (left->is_single_cpu()) {
1942     assert(left == dest, "left and dest must be equal");
1943     Register lreg = left->as_register();
1944 
1945     if (right->is_single_cpu()) {
1946       // cpu register - cpu register
1947       Register rreg = right->as_register();
1948       switch (code) {
1949         case lir_add: __ addl (lreg, rreg); break;
1950         case lir_sub: __ subl (lreg, rreg); break;
1951         case lir_mul: __ imull(lreg, rreg); break;
1952         default:      ShouldNotReachHere();
1953       }
1954 
1955     } else if (right->is_stack()) {
1956       // cpu register - stack
1957       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1958       switch (code) {
1959         case lir_add: __ addl(lreg, raddr); break;
1960         case lir_sub: __ subl(lreg, raddr); break;
1961         default:      ShouldNotReachHere();
1962       }
1963 
1964     } else if (right->is_constant()) {
1965       // cpu register - constant
1966       jint c = right->as_constant_ptr()->as_jint();
1967       switch (code) {
1968         case lir_add: {
1969           __ increment(lreg, c);
1970           break;
1971         }
1972         case lir_sub: {
1973           __ decrement(lreg, c);
1974           break;
1975         }
1976         default: ShouldNotReachHere();
1977       }
1978 
1979     } else {
1980       ShouldNotReachHere();
1981     }
1982 
1983   } else if (left->is_double_cpu()) {
1984     assert(left == dest, "left and dest must be equal");
1985     Register lreg_lo = left->as_register_lo();
1986     Register lreg_hi = left->as_register_hi();
1987 
1988     if (right->is_double_cpu()) {
1989       // cpu register - cpu register
1990       Register rreg_lo = right->as_register_lo();
1991       Register rreg_hi = right->as_register_hi();
1992       NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
1993       LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
1994       switch (code) {
1995         case lir_add:
1996           __ addptr(lreg_lo, rreg_lo);
1997           NOT_LP64(__ adcl(lreg_hi, rreg_hi));
1998           break;
1999         case lir_sub:
2000           __ subptr(lreg_lo, rreg_lo);
2001           NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2002           break;
2003         case lir_mul:
2004 #ifdef _LP64
2005           __ imulq(lreg_lo, rreg_lo);
2006 #else
2007           assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2008           __ imull(lreg_hi, rreg_lo);
2009           __ imull(rreg_hi, lreg_lo);
2010           __ addl (rreg_hi, lreg_hi);
2011           __ mull (rreg_lo);
2012           __ addl (lreg_hi, rreg_hi);
2013 #endif // _LP64
2014           break;
2015         default:
2016           ShouldNotReachHere();
2017       }
2018 
2019     } else if (right->is_constant()) {
2020       // cpu register - constant
2021 #ifdef _LP64
2022       jlong c = right->as_constant_ptr()->as_jlong_bits();
2023       __ movptr(r10, (intptr_t) c);
2024       switch (code) {
2025         case lir_add:
2026           __ addptr(lreg_lo, r10);
2027           break;
2028         case lir_sub:
2029           __ subptr(lreg_lo, r10);
2030           break;
2031         default:
2032           ShouldNotReachHere();
2033       }
2034 #else
2035       jint c_lo = right->as_constant_ptr()->as_jint_lo();
2036       jint c_hi = right->as_constant_ptr()->as_jint_hi();
2037       switch (code) {
2038         case lir_add:
2039           __ addptr(lreg_lo, c_lo);
2040           __ adcl(lreg_hi, c_hi);
2041           break;
2042         case lir_sub:
2043           __ subptr(lreg_lo, c_lo);
2044           __ sbbl(lreg_hi, c_hi);
2045           break;
2046         default:
2047           ShouldNotReachHere();
2048       }
2049 #endif // _LP64
2050 
2051     } else {
2052       ShouldNotReachHere();
2053     }
2054 
2055   } else if (left->is_single_xmm()) {
2056     assert(left == dest, "left and dest must be equal");
2057     XMMRegister lreg = left->as_xmm_float_reg();
2058 
2059     if (right->is_single_xmm()) {
2060       XMMRegister rreg = right->as_xmm_float_reg();
2061       switch (code) {
2062         case lir_add: __ addss(lreg, rreg);  break;
2063         case lir_sub: __ subss(lreg, rreg);  break;
2064         case lir_mul_strictfp: // fall through
2065         case lir_mul: __ mulss(lreg, rreg);  break;
2066         case lir_div_strictfp: // fall through
2067         case lir_div: __ divss(lreg, rreg);  break;
2068         default: ShouldNotReachHere();
2069       }
2070     } else {
2071       Address raddr;
2072       if (right->is_single_stack()) {
2073         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2074       } else if (right->is_constant()) {
2075         // hack for now
2076         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2077       } else {
2078         ShouldNotReachHere();
2079       }
2080       switch (code) {
2081         case lir_add: __ addss(lreg, raddr);  break;
2082         case lir_sub: __ subss(lreg, raddr);  break;
2083         case lir_mul_strictfp: // fall through
2084         case lir_mul: __ mulss(lreg, raddr);  break;
2085         case lir_div_strictfp: // fall through
2086         case lir_div: __ divss(lreg, raddr);  break;
2087         default: ShouldNotReachHere();
2088       }
2089     }
2090 
2091   } else if (left->is_double_xmm()) {
2092     assert(left == dest, "left and dest must be equal");
2093 
2094     XMMRegister lreg = left->as_xmm_double_reg();
2095     if (right->is_double_xmm()) {
2096       XMMRegister rreg = right->as_xmm_double_reg();
2097       switch (code) {
2098         case lir_add: __ addsd(lreg, rreg);  break;
2099         case lir_sub: __ subsd(lreg, rreg);  break;
2100         case lir_mul_strictfp: // fall through
2101         case lir_mul: __ mulsd(lreg, rreg);  break;
2102         case lir_div_strictfp: // fall through
2103         case lir_div: __ divsd(lreg, rreg);  break;
2104         default: ShouldNotReachHere();
2105       }
2106     } else {
2107       Address raddr;
2108       if (right->is_double_stack()) {
2109         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2110       } else if (right->is_constant()) {
2111         // hack for now
2112         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2113       } else {
2114         ShouldNotReachHere();
2115       }
2116       switch (code) {
2117         case lir_add: __ addsd(lreg, raddr);  break;
2118         case lir_sub: __ subsd(lreg, raddr);  break;
2119         case lir_mul_strictfp: // fall through
2120         case lir_mul: __ mulsd(lreg, raddr);  break;
2121         case lir_div_strictfp: // fall through
2122         case lir_div: __ divsd(lreg, raddr);  break;
2123         default: ShouldNotReachHere();
2124       }
2125     }
2126 
2127   } else if (left->is_single_fpu()) {
2128     assert(dest->is_single_fpu(),  "fpu stack allocation required");
2129 
2130     if (right->is_single_fpu()) {
2131       arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack);
2132 
2133     } else {
2134       assert(left->fpu_regnr() == 0, "left must be on TOS");
2135       assert(dest->fpu_regnr() == 0, "dest must be on TOS");
2136 
2137       Address raddr;
2138       if (right->is_single_stack()) {
2139         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2140       } else if (right->is_constant()) {
2141         address const_addr = float_constant(right->as_jfloat());
2142         assert(const_addr != NULL, "incorrect float/double constant maintainance");
2143         // hack for now
2144         raddr = __ as_Address(InternalAddress(const_addr));
2145       } else {
2146         ShouldNotReachHere();
2147       }
2148 
2149       switch (code) {
2150         case lir_add: __ fadd_s(raddr); break;
2151         case lir_sub: __ fsub_s(raddr); break;
2152         case lir_mul_strictfp: // fall through
2153         case lir_mul: __ fmul_s(raddr); break;
2154         case lir_div_strictfp: // fall through
2155         case lir_div: __ fdiv_s(raddr); break;
2156         default:      ShouldNotReachHere();
2157       }
2158     }
2159 
2160   } else if (left->is_double_fpu()) {
2161     assert(dest->is_double_fpu(),  "fpu stack allocation required");
2162 
2163     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2164       // Double values require special handling for strictfp mul/div on x86
2165       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
2166       __ fmulp(left->fpu_regnrLo() + 1);
2167     }
2168 
2169     if (right->is_double_fpu()) {
2170       arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack);
2171 
2172     } else {
2173       assert(left->fpu_regnrLo() == 0, "left must be on TOS");
2174       assert(dest->fpu_regnrLo() == 0, "dest must be on TOS");
2175 
2176       Address raddr;
2177       if (right->is_double_stack()) {
2178         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2179       } else if (right->is_constant()) {
2180         // hack for now
2181         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2182       } else {
2183         ShouldNotReachHere();
2184       }
2185 
2186       switch (code) {
2187         case lir_add: __ fadd_d(raddr); break;
2188         case lir_sub: __ fsub_d(raddr); break;
2189         case lir_mul_strictfp: // fall through
2190         case lir_mul: __ fmul_d(raddr); break;
2191         case lir_div_strictfp: // fall through
2192         case lir_div: __ fdiv_d(raddr); break;
2193         default: ShouldNotReachHere();
2194       }
2195     }
2196 
2197     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2198       // Double values require special handling for strictfp mul/div on x86
2199       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
2200       __ fmulp(dest->fpu_regnrLo() + 1);
2201     }
2202 
2203   } else if (left->is_single_stack() || left->is_address()) {
2204     assert(left == dest, "left and dest must be equal");
2205 
2206     Address laddr;
2207     if (left->is_single_stack()) {
2208       laddr = frame_map()->address_for_slot(left->single_stack_ix());
2209     } else if (left->is_address()) {
2210       laddr = as_Address(left->as_address_ptr());
2211     } else {
2212       ShouldNotReachHere();
2213     }
2214 
2215     if (right->is_single_cpu()) {
2216       Register rreg = right->as_register();
2217       switch (code) {
2218         case lir_add: __ addl(laddr, rreg); break;
2219         case lir_sub: __ subl(laddr, rreg); break;
2220         default:      ShouldNotReachHere();
2221       }
2222     } else if (right->is_constant()) {
2223       jint c = right->as_constant_ptr()->as_jint();
2224       switch (code) {
2225         case lir_add: {
2226           __ incrementl(laddr, c);
2227           break;
2228         }
2229         case lir_sub: {
2230           __ decrementl(laddr, c);
2231           break;
2232         }
2233         default: ShouldNotReachHere();
2234       }
2235     } else {
2236       ShouldNotReachHere();
2237     }
2238 
2239   } else {
2240     ShouldNotReachHere();
2241   }
2242 }
2243 
2244 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) {
2245   assert(pop_fpu_stack  || (left_index     == dest_index || right_index     == dest_index), "invalid LIR");
2246   assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR");
2247   assert(left_index == 0 || right_index == 0, "either must be on top of stack");
2248 
2249   bool left_is_tos = (left_index == 0);
2250   bool dest_is_tos = (dest_index == 0);
2251   int non_tos_index = (left_is_tos ? right_index : left_index);
2252 
2253   switch (code) {
2254     case lir_add:
2255       if (pop_fpu_stack)       __ faddp(non_tos_index);
2256       else if (dest_is_tos)    __ fadd (non_tos_index);
2257       else                     __ fadda(non_tos_index);
2258       break;
2259 
2260     case lir_sub:
2261       if (left_is_tos) {
2262         if (pop_fpu_stack)     __ fsubrp(non_tos_index);
2263         else if (dest_is_tos)  __ fsub  (non_tos_index);
2264         else                   __ fsubra(non_tos_index);
2265       } else {
2266         if (pop_fpu_stack)     __ fsubp (non_tos_index);
2267         else if (dest_is_tos)  __ fsubr (non_tos_index);
2268         else                   __ fsuba (non_tos_index);
2269       }
2270       break;
2271 
2272     case lir_mul_strictfp: // fall through
2273     case lir_mul:
2274       if (pop_fpu_stack)       __ fmulp(non_tos_index);
2275       else if (dest_is_tos)    __ fmul (non_tos_index);
2276       else                     __ fmula(non_tos_index);
2277       break;
2278 
2279     case lir_div_strictfp: // fall through
2280     case lir_div:
2281       if (left_is_tos) {
2282         if (pop_fpu_stack)     __ fdivrp(non_tos_index);
2283         else if (dest_is_tos)  __ fdiv  (non_tos_index);
2284         else                   __ fdivra(non_tos_index);
2285       } else {
2286         if (pop_fpu_stack)     __ fdivp (non_tos_index);
2287         else if (dest_is_tos)  __ fdivr (non_tos_index);
2288         else                   __ fdiva (non_tos_index);
2289       }
2290       break;
2291 
2292     case lir_rem:
2293       assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation");
2294       __ fremr(noreg);
2295       break;
2296 
2297     default:
2298       ShouldNotReachHere();
2299   }
2300 }
2301 
2302 
2303 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
2304   if (value->is_double_xmm()) {
2305     switch(code) {
2306       case lir_abs :
2307         {
2308           if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2309             __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2310           }
2311           __ andpd(dest->as_xmm_double_reg(),
2312                     ExternalAddress((address)double_signmask_pool));
2313         }
2314         break;
2315 
2316       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2317       // all other intrinsics are not available in the SSE instruction set, so FPU is used
2318       default      : ShouldNotReachHere();
2319     }
2320 
2321   } else if (value->is_double_fpu()) {
2322     assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2323     switch(code) {
2324       case lir_log   : __ flog() ; break;
2325       case lir_log10 : __ flog10() ; break;
2326       case lir_abs   : __ fabs() ; break;
2327       case lir_sqrt  : __ fsqrt(); break;
2328       case lir_sin   :
2329         // Should consider not saving rbx, if not necessary
2330         __ trigfunc('s', op->as_Op2()->fpu_stack_size());
2331         break;
2332       case lir_cos :
2333         // Should consider not saving rbx, if not necessary
2334         assert(op->as_Op2()->fpu_stack_size() <= 6, "sin and cos need two free stack slots");
2335         __ trigfunc('c', op->as_Op2()->fpu_stack_size());
2336         break;
2337       case lir_tan :
2338         // Should consider not saving rbx, if not necessary
2339         __ trigfunc('t', op->as_Op2()->fpu_stack_size());
2340         break;
2341       default      : ShouldNotReachHere();
2342     }
2343   } else {
2344     Unimplemented();
2345   }
2346 }
2347 
2348 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2349   // assert(left->destroys_register(), "check");
2350   if (left->is_single_cpu()) {
2351     Register reg = left->as_register();
2352     if (right->is_constant()) {
2353       int val = right->as_constant_ptr()->as_jint();
2354       switch (code) {
2355         case lir_logic_and: __ andl (reg, val); break;
2356         case lir_logic_or:  __ orl  (reg, val); break;
2357         case lir_logic_xor: __ xorl (reg, val); break;
2358         default: ShouldNotReachHere();
2359       }
2360     } else if (right->is_stack()) {
2361       // added support for stack operands
2362       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2363       switch (code) {
2364         case lir_logic_and: __ andl (reg, raddr); break;
2365         case lir_logic_or:  __ orl  (reg, raddr); break;
2366         case lir_logic_xor: __ xorl (reg, raddr); break;
2367         default: ShouldNotReachHere();
2368       }
2369     } else {
2370       Register rright = right->as_register();
2371       switch (code) {
2372         case lir_logic_and: __ andptr (reg, rright); break;
2373         case lir_logic_or : __ orptr  (reg, rright); break;
2374         case lir_logic_xor: __ xorptr (reg, rright); break;
2375         default: ShouldNotReachHere();
2376       }
2377     }
2378     move_regs(reg, dst->as_register());
2379   } else {
2380     Register l_lo = left->as_register_lo();
2381     Register l_hi = left->as_register_hi();
2382     if (right->is_constant()) {
2383 #ifdef _LP64
2384       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2385       switch (code) {
2386         case lir_logic_and:
2387           __ andq(l_lo, rscratch1);
2388           break;
2389         case lir_logic_or:
2390           __ orq(l_lo, rscratch1);
2391           break;
2392         case lir_logic_xor:
2393           __ xorq(l_lo, rscratch1);
2394           break;
2395         default: ShouldNotReachHere();
2396       }
2397 #else
2398       int r_lo = right->as_constant_ptr()->as_jint_lo();
2399       int r_hi = right->as_constant_ptr()->as_jint_hi();
2400       switch (code) {
2401         case lir_logic_and:
2402           __ andl(l_lo, r_lo);
2403           __ andl(l_hi, r_hi);
2404           break;
2405         case lir_logic_or:
2406           __ orl(l_lo, r_lo);
2407           __ orl(l_hi, r_hi);
2408           break;
2409         case lir_logic_xor:
2410           __ xorl(l_lo, r_lo);
2411           __ xorl(l_hi, r_hi);
2412           break;
2413         default: ShouldNotReachHere();
2414       }
2415 #endif // _LP64
2416     } else {
2417       Register r_lo = right->as_register_lo();
2418       Register r_hi = right->as_register_hi();
2419       assert(l_lo != r_hi, "overwriting registers");
2420       switch (code) {
2421         case lir_logic_and:
2422           __ andptr(l_lo, r_lo);
2423           NOT_LP64(__ andptr(l_hi, r_hi);)
2424           break;
2425         case lir_logic_or:
2426           __ orptr(l_lo, r_lo);
2427           NOT_LP64(__ orptr(l_hi, r_hi);)
2428           break;
2429         case lir_logic_xor:
2430           __ xorptr(l_lo, r_lo);
2431           NOT_LP64(__ xorptr(l_hi, r_hi);)
2432           break;
2433         default: ShouldNotReachHere();
2434       }
2435     }
2436 
2437     Register dst_lo = dst->as_register_lo();
2438     Register dst_hi = dst->as_register_hi();
2439 
2440 #ifdef _LP64
2441     move_regs(l_lo, dst_lo);
2442 #else
2443     if (dst_lo == l_hi) {
2444       assert(dst_hi != l_lo, "overwriting registers");
2445       move_regs(l_hi, dst_hi);
2446       move_regs(l_lo, dst_lo);
2447     } else {
2448       assert(dst_lo != l_hi, "overwriting registers");
2449       move_regs(l_lo, dst_lo);
2450       move_regs(l_hi, dst_hi);
2451     }
2452 #endif // _LP64
2453   }
2454 }
2455 
2456 
2457 // we assume that rax, and rdx can be overwritten
2458 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2459 
2460   assert(left->is_single_cpu(),   "left must be register");
2461   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2462   assert(result->is_single_cpu(), "result must be register");
2463 
2464   //  assert(left->destroys_register(), "check");
2465   //  assert(right->destroys_register(), "check");
2466 
2467   Register lreg = left->as_register();
2468   Register dreg = result->as_register();
2469 
2470   if (right->is_constant()) {
2471     int divisor = right->as_constant_ptr()->as_jint();
2472     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2473     if (code == lir_idiv) {
2474       assert(lreg == rax, "must be rax,");
2475       assert(temp->as_register() == rdx, "tmp register must be rdx");
2476       __ cdql(); // sign extend into rdx:rax
2477       if (divisor == 2) {
2478         __ subl(lreg, rdx);
2479       } else {
2480         __ andl(rdx, divisor - 1);
2481         __ addl(lreg, rdx);
2482       }
2483       __ sarl(lreg, log2_intptr(divisor));
2484       move_regs(lreg, dreg);
2485     } else if (code == lir_irem) {
2486       Label done;
2487       __ mov(dreg, lreg);
2488       __ andl(dreg, 0x80000000 | (divisor - 1));
2489       __ jcc(Assembler::positive, done);
2490       __ decrement(dreg);
2491       __ orl(dreg, ~(divisor - 1));
2492       __ increment(dreg);
2493       __ bind(done);
2494     } else {
2495       ShouldNotReachHere();
2496     }
2497   } else {
2498     Register rreg = right->as_register();
2499     assert(lreg == rax, "left register must be rax,");
2500     assert(rreg != rdx, "right register must not be rdx");
2501     assert(temp->as_register() == rdx, "tmp register must be rdx");
2502 
2503     move_regs(lreg, rax);
2504 
2505     int idivl_offset = __ corrected_idivl(rreg);
2506     add_debug_info_for_div0(idivl_offset, info);
2507     if (code == lir_irem) {
2508       move_regs(rdx, dreg); // result is in rdx
2509     } else {
2510       move_regs(rax, dreg);
2511     }
2512   }
2513 }
2514 
2515 
2516 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2517   if (opr1->is_single_cpu()) {
2518     Register reg1 = opr1->as_register();
2519     if (opr2->is_single_cpu()) {
2520       // cpu register - cpu register
2521       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2522         __ cmpptr(reg1, opr2->as_register());
2523       } else {
2524         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
2525         __ cmpl(reg1, opr2->as_register());
2526       }
2527     } else if (opr2->is_stack()) {
2528       // cpu register - stack
2529       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2530         __ cmpptr(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2531       } else {
2532         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2533       }
2534     } else if (opr2->is_constant()) {
2535       // cpu register - constant
2536       LIR_Const* c = opr2->as_constant_ptr();
2537       if (c->type() == T_INT) {
2538         __ cmpl(reg1, c->as_jint());
2539       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2540         // In 64bit oops are single register
2541         jobject o = c->as_jobject();
2542         if (o == NULL) {
2543           __ cmpptr(reg1, (int32_t)NULL_WORD);
2544         } else {
2545 #ifdef _LP64
2546           __ movoop(rscratch1, o);
2547           __ cmpptr(reg1, rscratch1);
2548 #else
2549           __ cmpoop(reg1, c->as_jobject());
2550 #endif // _LP64
2551         }
2552       } else {
2553         ShouldNotReachHere();
2554       }
2555       // cpu register - address
2556     } else if (opr2->is_address()) {
2557       if (op->info() != NULL) {
2558         add_debug_info_for_null_check_here(op->info());
2559       }
2560       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2561     } else {
2562       ShouldNotReachHere();
2563     }
2564 
2565   } else if(opr1->is_double_cpu()) {
2566     Register xlo = opr1->as_register_lo();
2567     Register xhi = opr1->as_register_hi();
2568     if (opr2->is_double_cpu()) {
2569 #ifdef _LP64
2570       __ cmpptr(xlo, opr2->as_register_lo());
2571 #else
2572       // cpu register - cpu register
2573       Register ylo = opr2->as_register_lo();
2574       Register yhi = opr2->as_register_hi();
2575       __ subl(xlo, ylo);
2576       __ sbbl(xhi, yhi);
2577       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2578         __ orl(xhi, xlo);
2579       }
2580 #endif // _LP64
2581     } else if (opr2->is_constant()) {
2582       // cpu register - constant 0
2583       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2584 #ifdef _LP64
2585       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2586 #else
2587       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2588       __ orl(xhi, xlo);
2589 #endif // _LP64
2590     } else {
2591       ShouldNotReachHere();
2592     }
2593 
2594   } else if (opr1->is_single_xmm()) {
2595     XMMRegister reg1 = opr1->as_xmm_float_reg();
2596     if (opr2->is_single_xmm()) {
2597       // xmm register - xmm register
2598       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2599     } else if (opr2->is_stack()) {
2600       // xmm register - stack
2601       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2602     } else if (opr2->is_constant()) {
2603       // xmm register - constant
2604       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2605     } else if (opr2->is_address()) {
2606       // xmm register - address
2607       if (op->info() != NULL) {
2608         add_debug_info_for_null_check_here(op->info());
2609       }
2610       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2611     } else {
2612       ShouldNotReachHere();
2613     }
2614 
2615   } else if (opr1->is_double_xmm()) {
2616     XMMRegister reg1 = opr1->as_xmm_double_reg();
2617     if (opr2->is_double_xmm()) {
2618       // xmm register - xmm register
2619       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2620     } else if (opr2->is_stack()) {
2621       // xmm register - stack
2622       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2623     } else if (opr2->is_constant()) {
2624       // xmm register - constant
2625       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2626     } else if (opr2->is_address()) {
2627       // xmm register - address
2628       if (op->info() != NULL) {
2629         add_debug_info_for_null_check_here(op->info());
2630       }
2631       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2632     } else {
2633       ShouldNotReachHere();
2634     }
2635 
2636   } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) {
2637     assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)");
2638     assert(opr2->is_fpu_register(), "both must be registers");
2639     __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2640 
2641   } else if (opr1->is_address() && opr2->is_constant()) {
2642     LIR_Const* c = opr2->as_constant_ptr();
2643 #ifdef _LP64
2644     if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2645       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2646       __ movoop(rscratch1, c->as_jobject());
2647     }
2648 #endif // LP64
2649     if (op->info() != NULL) {
2650       add_debug_info_for_null_check_here(op->info());
2651     }
2652     // special case: address - constant
2653     LIR_Address* addr = opr1->as_address_ptr();
2654     if (c->type() == T_INT) {
2655       __ cmpl(as_Address(addr), c->as_jint());
2656     } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2657 #ifdef _LP64
2658       // %%% Make this explode if addr isn't reachable until we figure out a
2659       // better strategy by giving noreg as the temp for as_Address
2660       __ cmpptr(rscratch1, as_Address(addr, noreg));
2661 #else
2662       __ cmpoop(as_Address(addr), c->as_jobject());
2663 #endif // _LP64
2664     } else {
2665       ShouldNotReachHere();
2666     }
2667 
2668   } else {
2669     ShouldNotReachHere();
2670   }
2671 }
2672 
2673 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2674   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2675     if (left->is_single_xmm()) {
2676       assert(right->is_single_xmm(), "must match");
2677       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2678     } else if (left->is_double_xmm()) {
2679       assert(right->is_double_xmm(), "must match");
2680       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2681 
2682     } else {
2683       assert(left->is_single_fpu() || left->is_double_fpu(), "must be");
2684       assert(right->is_single_fpu() || right->is_double_fpu(), "must match");
2685 
2686       assert(left->fpu() == 0, "left must be on TOS");
2687       __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2688                   op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2689     }
2690   } else {
2691     assert(code == lir_cmp_l2i, "check");
2692 #ifdef _LP64
2693     Label done;
2694     Register dest = dst->as_register();
2695     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2696     __ movl(dest, -1);
2697     __ jccb(Assembler::less, done);
2698     __ set_byte_if_not_zero(dest);
2699     __ movzbl(dest, dest);
2700     __ bind(done);
2701 #else
2702     __ lcmp2int(left->as_register_hi(),
2703                 left->as_register_lo(),
2704                 right->as_register_hi(),
2705                 right->as_register_lo());
2706     move_regs(left->as_register_hi(), dst->as_register());
2707 #endif // _LP64
2708   }
2709 }
2710 
2711 
2712 void LIR_Assembler::align_call(LIR_Code code) {
2713   if (os::is_MP()) {
2714     // make sure that the displacement word of the call ends up word aligned
2715     int offset = __ offset();
2716     switch (code) {
2717       case lir_static_call:
2718       case lir_optvirtual_call:
2719       case lir_dynamic_call:
2720         offset += NativeCall::displacement_offset;
2721         break;
2722       case lir_icvirtual_call:
2723         offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
2724       break;
2725       case lir_virtual_call:  // currently, sparc-specific for niagara
2726       default: ShouldNotReachHere();
2727     }
2728     while (offset++ % BytesPerWord != 0) {
2729       __ nop();
2730     }
2731   }
2732 }
2733 
2734 
2735 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2736   assert(!os::is_MP() || (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2737          "must be aligned");
2738   __ call(AddressLiteral(op->addr(), rtype));
2739   add_call_info(code_offset(), op->info(), op->is_method_handle_invoke());
2740 }
2741 
2742 
2743 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2744   RelocationHolder rh = virtual_call_Relocation::spec(pc());
2745   __ movoop(IC_Klass, (jobject)Universe::non_oop_word());
2746   assert(!os::is_MP() ||
2747          (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2748          "must be aligned");
2749   __ call(AddressLiteral(op->addr(), rh));
2750   add_call_info(code_offset(), op->info(), op->is_method_handle_invoke());
2751 }
2752 
2753 
2754 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2755 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2756   ShouldNotReachHere();
2757 }
2758 
2759 
2760 void LIR_Assembler::preserve_SP(LIR_OpJavaCall* op) {
2761   __ movptr(FrameMap::method_handle_invoke_SP_save_opr()->as_register(), rsp);
2762 }
2763 
2764 
2765 void LIR_Assembler::restore_SP(LIR_OpJavaCall* op) {
2766   __ movptr(rsp, FrameMap::method_handle_invoke_SP_save_opr()->as_register());
2767 }
2768 
2769 
2770 void LIR_Assembler::emit_static_call_stub() {
2771   address call_pc = __ pc();
2772   address stub = __ start_a_stub(call_stub_size);
2773   if (stub == NULL) {
2774     bailout("static call stub overflow");
2775     return;
2776   }
2777 
2778   int start = __ offset();
2779   if (os::is_MP()) {
2780     // make sure that the displacement word of the call ends up word aligned
2781     int offset = __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset;
2782     while (offset++ % BytesPerWord != 0) {
2783       __ nop();
2784     }
2785   }
2786   __ relocate(static_stub_Relocation::spec(call_pc));
2787   __ movoop(rbx, (jobject)NULL);
2788   // must be set to -1 at code generation time
2789   assert(!os::is_MP() || ((__ offset() + 1) % BytesPerWord) == 0, "must be aligned on MP");
2790   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2791   __ jump(RuntimeAddress(__ pc()));
2792 
2793   assert(__ offset() - start <= call_stub_size, "stub too big")
2794   __ end_a_stub();
2795 }
2796 
2797 
2798 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info, bool unwind) {
2799   assert(exceptionOop->as_register() == rax, "must match");
2800   assert(unwind || exceptionPC->as_register() == rdx, "must match");
2801 
2802   // exception object is not added to oop map by LinearScan
2803   // (LinearScan assumes that no oops are in fixed registers)
2804   info->add_register_oop(exceptionOop);
2805   Runtime1::StubID unwind_id;
2806 
2807   if (!unwind) {
2808     // get current pc information
2809     // pc is only needed if the method has an exception handler, the unwind code does not need it.
2810     int pc_for_athrow_offset = __ offset();
2811     InternalAddress pc_for_athrow(__ pc());
2812     __ lea(exceptionPC->as_register(), pc_for_athrow);
2813     add_call_info(pc_for_athrow_offset, info); // for exception handler
2814 
2815     __ verify_not_null_oop(rax);
2816     // search an exception handler (rax: exception oop, rdx: throwing pc)
2817     if (compilation()->has_fpu_code()) {
2818       unwind_id = Runtime1::handle_exception_id;
2819     } else {
2820       unwind_id = Runtime1::handle_exception_nofpu_id;
2821     }
2822     __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2823   } else {
2824     // remove the activation
2825     __ remove_frame(initial_frame_size_in_bytes());
2826     __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
2827   }
2828 
2829   // enough room for two byte trap
2830   __ nop();
2831 }
2832 
2833 
2834 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2835 
2836   // optimized version for linear scan:
2837   // * count must be already in ECX (guaranteed by LinearScan)
2838   // * left and dest must be equal
2839   // * tmp must be unused
2840   assert(count->as_register() == SHIFT_count, "count must be in ECX");
2841   assert(left == dest, "left and dest must be equal");
2842   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2843 
2844   if (left->is_single_cpu()) {
2845     Register value = left->as_register();
2846     assert(value != SHIFT_count, "left cannot be ECX");
2847 
2848     switch (code) {
2849       case lir_shl:  __ shll(value); break;
2850       case lir_shr:  __ sarl(value); break;
2851       case lir_ushr: __ shrl(value); break;
2852       default: ShouldNotReachHere();
2853     }
2854   } else if (left->is_double_cpu()) {
2855     Register lo = left->as_register_lo();
2856     Register hi = left->as_register_hi();
2857     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2858 #ifdef _LP64
2859     switch (code) {
2860       case lir_shl:  __ shlptr(lo);        break;
2861       case lir_shr:  __ sarptr(lo);        break;
2862       case lir_ushr: __ shrptr(lo);        break;
2863       default: ShouldNotReachHere();
2864     }
2865 #else
2866 
2867     switch (code) {
2868       case lir_shl:  __ lshl(hi, lo);        break;
2869       case lir_shr:  __ lshr(hi, lo, true);  break;
2870       case lir_ushr: __ lshr(hi, lo, false); break;
2871       default: ShouldNotReachHere();
2872     }
2873 #endif // LP64
2874   } else {
2875     ShouldNotReachHere();
2876   }
2877 }
2878 
2879 
2880 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2881   if (dest->is_single_cpu()) {
2882     // first move left into dest so that left is not destroyed by the shift
2883     Register value = dest->as_register();
2884     count = count & 0x1F; // Java spec
2885 
2886     move_regs(left->as_register(), value);
2887     switch (code) {
2888       case lir_shl:  __ shll(value, count); break;
2889       case lir_shr:  __ sarl(value, count); break;
2890       case lir_ushr: __ shrl(value, count); break;
2891       default: ShouldNotReachHere();
2892     }
2893   } else if (dest->is_double_cpu()) {
2894 #ifndef _LP64
2895     Unimplemented();
2896 #else
2897     // first move left into dest so that left is not destroyed by the shift
2898     Register value = dest->as_register_lo();
2899     count = count & 0x1F; // Java spec
2900 
2901     move_regs(left->as_register_lo(), value);
2902     switch (code) {
2903       case lir_shl:  __ shlptr(value, count); break;
2904       case lir_shr:  __ sarptr(value, count); break;
2905       case lir_ushr: __ shrptr(value, count); break;
2906       default: ShouldNotReachHere();
2907     }
2908 #endif // _LP64
2909   } else {
2910     ShouldNotReachHere();
2911   }
2912 }
2913 
2914 
2915 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2916   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2917   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2918   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2919   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2920 }
2921 
2922 
2923 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2924   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2925   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2926   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2927   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2928 }
2929 
2930 
2931 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2932   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2933   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2934   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2935   __ movoop (Address(rsp, offset_from_rsp_in_bytes), o);
2936 }
2937 
2938 
2939 // This code replaces a call to arraycopy; no exception may
2940 // be thrown in this code, they must be thrown in the System.arraycopy
2941 // activation frame; we could save some checks if this would not be the case
2942 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2943   ciArrayKlass* default_type = op->expected_type();
2944   Register src = op->src()->as_register();
2945   Register dst = op->dst()->as_register();
2946   Register src_pos = op->src_pos()->as_register();
2947   Register dst_pos = op->dst_pos()->as_register();
2948   Register length  = op->length()->as_register();
2949   Register tmp = op->tmp()->as_register();
2950 
2951   CodeStub* stub = op->stub();
2952   int flags = op->flags();
2953   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2954   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2955 
2956   // if we don't know anything or it's an object array, just go through the generic arraycopy
2957   if (default_type == NULL) {
2958     Label done;
2959     // save outgoing arguments on stack in case call to System.arraycopy is needed
2960     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2961     // for interpreter calling conventions. Now we have to do it in new style conventions.
2962     // For the moment until C1 gets the new register allocator I just force all the
2963     // args to the right place (except the register args) and then on the back side
2964     // reload the register args properly if we go slow path. Yuck
2965 
2966     // These are proper for the calling convention
2967 
2968     store_parameter(length, 2);
2969     store_parameter(dst_pos, 1);
2970     store_parameter(dst, 0);
2971 
2972     // these are just temporary placements until we need to reload
2973     store_parameter(src_pos, 3);
2974     store_parameter(src, 4);
2975     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
2976 
2977     address entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2978 
2979     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2980 #ifdef _LP64
2981     // The arguments are in java calling convention so we can trivially shift them to C
2982     // convention
2983     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2984     __ mov(c_rarg0, j_rarg0);
2985     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2986     __ mov(c_rarg1, j_rarg1);
2987     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2988     __ mov(c_rarg2, j_rarg2);
2989     assert_different_registers(c_rarg3, j_rarg4);
2990     __ mov(c_rarg3, j_rarg3);
2991 #ifdef _WIN64
2992     // Allocate abi space for args but be sure to keep stack aligned
2993     __ subptr(rsp, 6*wordSize);
2994     store_parameter(j_rarg4, 4);
2995     __ call(RuntimeAddress(entry));
2996     __ addptr(rsp, 6*wordSize);
2997 #else
2998     __ mov(c_rarg4, j_rarg4);
2999     __ call(RuntimeAddress(entry));
3000 #endif // _WIN64
3001 #else
3002     __ push(length);
3003     __ push(dst_pos);
3004     __ push(dst);
3005     __ push(src_pos);
3006     __ push(src);
3007     __ call_VM_leaf(entry, 5); // removes pushed parameter from the stack
3008 
3009 #endif // _LP64
3010 
3011     __ cmpl(rax, 0);
3012     __ jcc(Assembler::equal, *stub->continuation());
3013 
3014     // Reload values from the stack so they are where the stub
3015     // expects them.
3016     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3017     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3018     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3019     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3020     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3021     __ jmp(*stub->entry());
3022 
3023     __ bind(*stub->continuation());
3024     return;
3025   }
3026 
3027   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3028 
3029   int elem_size = type2aelembytes(basic_type);
3030   int shift_amount;
3031   Address::ScaleFactor scale;
3032 
3033   switch (elem_size) {
3034     case 1 :
3035       shift_amount = 0;
3036       scale = Address::times_1;
3037       break;
3038     case 2 :
3039       shift_amount = 1;
3040       scale = Address::times_2;
3041       break;
3042     case 4 :
3043       shift_amount = 2;
3044       scale = Address::times_4;
3045       break;
3046     case 8 :
3047       shift_amount = 3;
3048       scale = Address::times_8;
3049       break;
3050     default:
3051       ShouldNotReachHere();
3052   }
3053 
3054   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3055   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3056   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3057   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3058 
3059   // length and pos's are all sign extended at this point on 64bit
3060 
3061   // test for NULL
3062   if (flags & LIR_OpArrayCopy::src_null_check) {
3063     __ testptr(src, src);
3064     __ jcc(Assembler::zero, *stub->entry());
3065   }
3066   if (flags & LIR_OpArrayCopy::dst_null_check) {
3067     __ testptr(dst, dst);
3068     __ jcc(Assembler::zero, *stub->entry());
3069   }
3070 
3071   // check if negative
3072   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3073     __ testl(src_pos, src_pos);
3074     __ jcc(Assembler::less, *stub->entry());
3075   }
3076   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3077     __ testl(dst_pos, dst_pos);
3078     __ jcc(Assembler::less, *stub->entry());
3079   }
3080   if (flags & LIR_OpArrayCopy::length_positive_check) {
3081     __ testl(length, length);
3082     __ jcc(Assembler::less, *stub->entry());
3083   }
3084 
3085   if (flags & LIR_OpArrayCopy::src_range_check) {
3086     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3087     __ cmpl(tmp, src_length_addr);
3088     __ jcc(Assembler::above, *stub->entry());
3089   }
3090   if (flags & LIR_OpArrayCopy::dst_range_check) {
3091     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3092     __ cmpl(tmp, dst_length_addr);
3093     __ jcc(Assembler::above, *stub->entry());
3094   }
3095 
3096   if (flags & LIR_OpArrayCopy::type_check) {
3097     __ movptr(tmp, src_klass_addr);
3098     __ cmpptr(tmp, dst_klass_addr);
3099     __ jcc(Assembler::notEqual, *stub->entry());
3100   }
3101 
3102 #ifdef ASSERT
3103   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3104     // Sanity check the known type with the incoming class.  For the
3105     // primitive case the types must match exactly with src.klass and
3106     // dst.klass each exactly matching the default type.  For the
3107     // object array case, if no type check is needed then either the
3108     // dst type is exactly the expected type and the src type is a
3109     // subtype which we can't check or src is the same array as dst
3110     // but not necessarily exactly of type default_type.
3111     Label known_ok, halt;
3112     __ movoop(tmp, default_type->constant_encoding());
3113     if (basic_type != T_OBJECT) {
3114       __ cmpptr(tmp, dst_klass_addr);
3115       __ jcc(Assembler::notEqual, halt);
3116       __ cmpptr(tmp, src_klass_addr);
3117       __ jcc(Assembler::equal, known_ok);
3118     } else {
3119       __ cmpptr(tmp, dst_klass_addr);
3120       __ jcc(Assembler::equal, known_ok);
3121       __ cmpptr(src, dst);
3122       __ jcc(Assembler::equal, known_ok);
3123     }
3124     __ bind(halt);
3125     __ stop("incorrect type information in arraycopy");
3126     __ bind(known_ok);
3127   }
3128 #endif
3129 
3130   if (shift_amount > 0 && basic_type != T_OBJECT) {
3131     __ shlptr(length, shift_amount);
3132   }
3133 
3134 #ifdef _LP64
3135   assert_different_registers(c_rarg0, dst, dst_pos, length);
3136   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3137   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3138   assert_different_registers(c_rarg1, length);
3139   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3140   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3141   __ mov(c_rarg2, length);
3142 
3143 #else
3144   __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3145   store_parameter(tmp, 0);
3146   __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3147   store_parameter(tmp, 1);
3148   store_parameter(length, 2);
3149 #endif // _LP64
3150   if (basic_type == T_OBJECT) {
3151     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Runtime1::oop_arraycopy), 0);
3152   } else {
3153     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Runtime1::primitive_arraycopy), 0);
3154   }
3155 
3156   __ bind(*stub->continuation());
3157 }
3158 
3159 
3160 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3161   Register obj = op->obj_opr()->as_register();  // may not be an oop
3162   Register hdr = op->hdr_opr()->as_register();
3163   Register lock = op->lock_opr()->as_register();
3164   if (!UseFastLocking) {
3165     __ jmp(*op->stub()->entry());
3166   } else if (op->code() == lir_lock) {
3167     Register scratch = noreg;
3168     if (UseBiasedLocking) {
3169       scratch = op->scratch_opr()->as_register();
3170     }
3171     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3172     // add debug info for NullPointerException only if one is possible
3173     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
3174     if (op->info() != NULL) {
3175       add_debug_info_for_null_check(null_check_offset, op->info());
3176     }
3177     // done
3178   } else if (op->code() == lir_unlock) {
3179     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3180     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3181   } else {
3182     Unimplemented();
3183   }
3184   __ bind(*op->stub()->continuation());
3185 }
3186 
3187 
3188 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3189   ciMethod* method = op->profiled_method();
3190   int bci          = op->profiled_bci();
3191 
3192   // Update counter for all call types
3193   ciMethodData* md = method->method_data();
3194   if (md == NULL) {
3195     bailout("out of memory building methodDataOop");
3196     return;
3197   }
3198   ciProfileData* data = md->bci_to_data(bci);
3199   assert(data->is_CounterData(), "need CounterData for calls");
3200   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3201   Register mdo  = op->mdo()->as_register();
3202   __ movoop(mdo, md->constant_encoding());
3203   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3204   Bytecodes::Code bc = method->java_code_at_bci(bci);
3205   // Perform additional virtual call profiling for invokevirtual and
3206   // invokeinterface bytecodes
3207   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
3208       Tier1ProfileVirtualCalls) {
3209     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3210     Register recv = op->recv()->as_register();
3211     assert_different_registers(mdo, recv);
3212     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3213     ciKlass* known_klass = op->known_holder();
3214     if (Tier1OptimizeVirtualCallProfiling && known_klass != NULL) {
3215       // We know the type that will be seen at this call site; we can
3216       // statically update the methodDataOop rather than needing to do
3217       // dynamic tests on the receiver type
3218 
3219       // NOTE: we should probably put a lock around this search to
3220       // avoid collisions by concurrent compilations
3221       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3222       uint i;
3223       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3224         ciKlass* receiver = vc_data->receiver(i);
3225         if (known_klass->equals(receiver)) {
3226           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3227           __ addl(data_addr, DataLayout::counter_increment);
3228           return;
3229         }
3230       }
3231 
3232       // Receiver type not found in profile data; select an empty slot
3233 
3234       // Note that this is less efficient than it should be because it
3235       // always does a write to the receiver part of the
3236       // VirtualCallData rather than just the first time
3237       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3238         ciKlass* receiver = vc_data->receiver(i);
3239         if (receiver == NULL) {
3240           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3241           __ movoop(recv_addr, known_klass->constant_encoding());
3242           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3243           __ addl(data_addr, DataLayout::counter_increment);
3244           return;
3245         }
3246       }
3247     } else {
3248       __ movptr(recv, Address(recv, oopDesc::klass_offset_in_bytes()));
3249       Label update_done;
3250       uint i;
3251       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3252         Label next_test;
3253         // See if the receiver is receiver[n].
3254         __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))));
3255         __ jcc(Assembler::notEqual, next_test);
3256         Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3257         __ addl(data_addr, DataLayout::counter_increment);
3258         __ jmp(update_done);
3259         __ bind(next_test);
3260       }
3261 
3262       // Didn't find receiver; find next empty slot and fill it in
3263       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3264         Label next_test;
3265         Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3266         __ cmpptr(recv_addr, (int32_t)NULL_WORD);
3267         __ jcc(Assembler::notEqual, next_test);
3268         __ movptr(recv_addr, recv);
3269         __ movl(Address(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))), DataLayout::counter_increment);
3270         __ jmp(update_done);
3271         __ bind(next_test);
3272       }
3273       // Receiver did not match any saved receiver and there is no empty row for it.
3274       // Increment total counter to indicate polymorphic case.
3275       __ addl(counter_addr, DataLayout::counter_increment);
3276 
3277       __ bind(update_done);
3278     }
3279   } else {
3280     // Static call
3281     __ addl(counter_addr, DataLayout::counter_increment);
3282   }
3283 }
3284 
3285 
3286 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3287   Unimplemented();
3288 }
3289 
3290 
3291 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3292   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3293 }
3294 
3295 
3296 void LIR_Assembler::align_backward_branch_target() {
3297   __ align(BytesPerWord);
3298 }
3299 
3300 
3301 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
3302   if (left->is_single_cpu()) {
3303     __ negl(left->as_register());
3304     move_regs(left->as_register(), dest->as_register());
3305 
3306   } else if (left->is_double_cpu()) {
3307     Register lo = left->as_register_lo();
3308 #ifdef _LP64
3309     Register dst = dest->as_register_lo();
3310     __ movptr(dst, lo);
3311     __ negptr(dst);
3312 #else
3313     Register hi = left->as_register_hi();
3314     __ lneg(hi, lo);
3315     if (dest->as_register_lo() == hi) {
3316       assert(dest->as_register_hi() != lo, "destroying register");
3317       move_regs(hi, dest->as_register_hi());
3318       move_regs(lo, dest->as_register_lo());
3319     } else {
3320       move_regs(lo, dest->as_register_lo());
3321       move_regs(hi, dest->as_register_hi());
3322     }
3323 #endif // _LP64
3324 
3325   } else if (dest->is_single_xmm()) {
3326     if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3327       __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3328     }
3329     __ xorps(dest->as_xmm_float_reg(),
3330              ExternalAddress((address)float_signflip_pool));
3331 
3332   } else if (dest->is_double_xmm()) {
3333     if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3334       __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3335     }
3336     __ xorpd(dest->as_xmm_double_reg(),
3337              ExternalAddress((address)double_signflip_pool));
3338 
3339   } else if (left->is_single_fpu() || left->is_double_fpu()) {
3340     assert(left->fpu() == 0, "arg must be on TOS");
3341     assert(dest->fpu() == 0, "dest must be TOS");
3342     __ fchs();
3343 
3344   } else {
3345     ShouldNotReachHere();
3346   }
3347 }
3348 
3349 
3350 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
3351   assert(addr->is_address() && dest->is_register(), "check");
3352   Register reg;
3353   reg = dest->as_pointer_register();
3354   __ lea(reg, as_Address(addr->as_address_ptr()));
3355 }
3356 
3357 
3358 
3359 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3360   assert(!tmp->is_valid(), "don't need temporary");
3361   __ call(RuntimeAddress(dest));
3362   if (info != NULL) {
3363     add_call_info_here(info);
3364   }
3365 }
3366 
3367 
3368 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3369   assert(type == T_LONG, "only for volatile long fields");
3370 
3371   if (info != NULL) {
3372     add_debug_info_for_null_check_here(info);
3373   }
3374 
3375   if (src->is_double_xmm()) {
3376     if (dest->is_double_cpu()) {
3377 #ifdef _LP64
3378       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3379 #else
3380       __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
3381       __ psrlq(src->as_xmm_double_reg(), 32);
3382       __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
3383 #endif // _LP64
3384     } else if (dest->is_double_stack()) {
3385       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3386     } else if (dest->is_address()) {
3387       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3388     } else {
3389       ShouldNotReachHere();
3390     }
3391 
3392   } else if (dest->is_double_xmm()) {
3393     if (src->is_double_stack()) {
3394       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3395     } else if (src->is_address()) {
3396       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3397     } else {
3398       ShouldNotReachHere();
3399     }
3400 
3401   } else if (src->is_double_fpu()) {
3402     assert(src->fpu_regnrLo() == 0, "must be TOS");
3403     if (dest->is_double_stack()) {
3404       __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
3405     } else if (dest->is_address()) {
3406       __ fistp_d(as_Address(dest->as_address_ptr()));
3407     } else {
3408       ShouldNotReachHere();
3409     }
3410 
3411   } else if (dest->is_double_fpu()) {
3412     assert(dest->fpu_regnrLo() == 0, "must be TOS");
3413     if (src->is_double_stack()) {
3414       __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
3415     } else if (src->is_address()) {
3416       __ fild_d(as_Address(src->as_address_ptr()));
3417     } else {
3418       ShouldNotReachHere();
3419     }
3420   } else {
3421     ShouldNotReachHere();
3422   }
3423 }
3424 
3425 
3426 void LIR_Assembler::membar() {
3427   // QQQ sparc TSO uses this,
3428   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3429 }
3430 
3431 void LIR_Assembler::membar_acquire() {
3432   // No x86 machines currently require load fences
3433   // __ load_fence();
3434 }
3435 
3436 void LIR_Assembler::membar_release() {
3437   // No x86 machines currently require store fences
3438   // __ store_fence();
3439 }
3440 
3441 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3442   assert(result_reg->is_register(), "check");
3443 #ifdef _LP64
3444   // __ get_thread(result_reg->as_register_lo());
3445   __ mov(result_reg->as_register(), r15_thread);
3446 #else
3447   __ get_thread(result_reg->as_register());
3448 #endif // _LP64
3449 }
3450 
3451 
3452 void LIR_Assembler::peephole(LIR_List*) {
3453   // do nothing for now
3454 }
3455 
3456 
3457 #undef __