1 /*
   2  * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "c1/c1_Compilation.hpp"
  27 #include "c1/c1_LIRAssembler.hpp"
  28 #include "c1/c1_MacroAssembler.hpp"
  29 #include "c1/c1_Runtime1.hpp"
  30 #include "c1/c1_ValueStack.hpp"
  31 #include "ci/ciArrayKlass.hpp"
  32 #include "ci/ciInstance.hpp"
  33 #include "gc_interface/collectedHeap.hpp"
  34 #include "memory/barrierSet.hpp"
  35 #include "memory/cardTableModRefBS.hpp"
  36 #include "nativeInst_x86.hpp"
  37 #include "oops/objArrayKlass.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 
  40 
  41 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  42 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  43 // fast versions of NegF/NegD and AbsF/AbsD.
  44 
  45 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  46 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  47   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  48   // of 128-bits operands for SSE instructions.
  49   jlong *operand = (jlong*)(((long)adr)&((long)(~0xF)));
  50   // Store the value to a 128-bits operand.
  51   operand[0] = lo;
  52   operand[1] = hi;
  53   return operand;
  54 }
  55 
  56 // Buffer for 128-bits masks used by SSE instructions.
  57 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  58 
  59 // Static initialization during VM startup.
  60 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
  61 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
  62 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));
  63 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
  64 
  65 
  66 
  67 NEEDS_CLEANUP // remove this definitions ?
  68 const Register IC_Klass    = rax;   // where the IC klass is cached
  69 const Register SYNC_header = rax;   // synchronization header
  70 const Register SHIFT_count = rcx;   // where count for shift operations must be
  71 
  72 #define __ _masm->
  73 
  74 
  75 static void select_different_registers(Register preserve,
  76                                        Register extra,
  77                                        Register &tmp1,
  78                                        Register &tmp2) {
  79   if (tmp1 == preserve) {
  80     assert_different_registers(tmp1, tmp2, extra);
  81     tmp1 = extra;
  82   } else if (tmp2 == preserve) {
  83     assert_different_registers(tmp1, tmp2, extra);
  84     tmp2 = extra;
  85   }
  86   assert_different_registers(preserve, tmp1, tmp2);
  87 }
  88 
  89 
  90 
  91 static void select_different_registers(Register preserve,
  92                                        Register extra,
  93                                        Register &tmp1,
  94                                        Register &tmp2,
  95                                        Register &tmp3) {
  96   if (tmp1 == preserve) {
  97     assert_different_registers(tmp1, tmp2, tmp3, extra);
  98     tmp1 = extra;
  99   } else if (tmp2 == preserve) {
 100     assert_different_registers(tmp1, tmp2, tmp3, extra);
 101     tmp2 = extra;
 102   } else if (tmp3 == preserve) {
 103     assert_different_registers(tmp1, tmp2, tmp3, extra);
 104     tmp3 = extra;
 105   }
 106   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 107 }
 108 
 109 
 110 
 111 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 112   if (opr->is_constant()) {
 113     LIR_Const* constant = opr->as_constant_ptr();
 114     switch (constant->type()) {
 115       case T_INT: {
 116         return true;
 117       }
 118 
 119       default:
 120         return false;
 121     }
 122   }
 123   return false;
 124 }
 125 
 126 
 127 LIR_Opr LIR_Assembler::receiverOpr() {
 128   return FrameMap::receiver_opr;
 129 }
 130 
 131 LIR_Opr LIR_Assembler::incomingReceiverOpr() {
 132   return receiverOpr();
 133 }
 134 
 135 LIR_Opr LIR_Assembler::osrBufferPointer() {
 136   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 137 }
 138 
 139 //--------------fpu register translations-----------------------
 140 
 141 
 142 address LIR_Assembler::float_constant(float f) {
 143   address const_addr = __ float_constant(f);
 144   if (const_addr == NULL) {
 145     bailout("const section overflow");
 146     return __ code()->consts()->start();
 147   } else {
 148     return const_addr;
 149   }
 150 }
 151 
 152 
 153 address LIR_Assembler::double_constant(double d) {
 154   address const_addr = __ double_constant(d);
 155   if (const_addr == NULL) {
 156     bailout("const section overflow");
 157     return __ code()->consts()->start();
 158   } else {
 159     return const_addr;
 160   }
 161 }
 162 
 163 
 164 void LIR_Assembler::set_24bit_FPU() {
 165   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
 166 }
 167 
 168 void LIR_Assembler::reset_FPU() {
 169   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 170 }
 171 
 172 void LIR_Assembler::fpop() {
 173   __ fpop();
 174 }
 175 
 176 void LIR_Assembler::fxch(int i) {
 177   __ fxch(i);
 178 }
 179 
 180 void LIR_Assembler::fld(int i) {
 181   __ fld_s(i);
 182 }
 183 
 184 void LIR_Assembler::ffree(int i) {
 185   __ ffree(i);
 186 }
 187 
 188 void LIR_Assembler::breakpoint() {
 189   __ int3();
 190 }
 191 
 192 void LIR_Assembler::push(LIR_Opr opr) {
 193   if (opr->is_single_cpu()) {
 194     __ push_reg(opr->as_register());
 195   } else if (opr->is_double_cpu()) {
 196     NOT_LP64(__ push_reg(opr->as_register_hi()));
 197     __ push_reg(opr->as_register_lo());
 198   } else if (opr->is_stack()) {
 199     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 200   } else if (opr->is_constant()) {
 201     LIR_Const* const_opr = opr->as_constant_ptr();
 202     if (const_opr->type() == T_OBJECT) {
 203       __ push_oop(const_opr->as_jobject());
 204     } else if (const_opr->type() == T_INT) {
 205       __ push_jint(const_opr->as_jint());
 206     } else {
 207       ShouldNotReachHere();
 208     }
 209 
 210   } else {
 211     ShouldNotReachHere();
 212   }
 213 }
 214 
 215 void LIR_Assembler::pop(LIR_Opr opr) {
 216   if (opr->is_single_cpu()) {
 217     __ pop_reg(opr->as_register());
 218   } else {
 219     ShouldNotReachHere();
 220   }
 221 }
 222 
 223 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 224   return addr->base()->is_illegal() && addr->index()->is_illegal();
 225 }
 226 
 227 //-------------------------------------------
 228 
 229 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 230   return as_Address(addr, rscratch1);
 231 }
 232 
 233 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 234   if (addr->base()->is_illegal()) {
 235     assert(addr->index()->is_illegal(), "must be illegal too");
 236     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 237     if (! __ reachable(laddr)) {
 238       __ movptr(tmp, laddr.addr());
 239       Address res(tmp, 0);
 240       return res;
 241     } else {
 242       return __ as_Address(laddr);
 243     }
 244   }
 245 
 246   Register base = addr->base()->as_pointer_register();
 247 
 248   if (addr->index()->is_illegal()) {
 249     return Address( base, addr->disp());
 250   } else if (addr->index()->is_cpu_register()) {
 251     Register index = addr->index()->as_pointer_register();
 252     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 253   } else if (addr->index()->is_constant()) {
 254     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 255     assert(Assembler::is_simm32(addr_offset), "must be");
 256 
 257     return Address(base, addr_offset);
 258   } else {
 259     Unimplemented();
 260     return Address();
 261   }
 262 }
 263 
 264 
 265 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 266   Address base = as_Address(addr);
 267   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 268 }
 269 
 270 
 271 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 272   return as_Address(addr);
 273 }
 274 
 275 
 276 void LIR_Assembler::osr_entry() {
 277   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 278   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 279   ValueStack* entry_state = osr_entry->state();
 280   int number_of_locks = entry_state->locks_size();
 281 
 282   // we jump here if osr happens with the interpreter
 283   // state set up to continue at the beginning of the
 284   // loop that triggered osr - in particular, we have
 285   // the following registers setup:
 286   //
 287   // rcx: osr buffer
 288   //
 289 
 290   // build frame
 291   ciMethod* m = compilation()->method();
 292   __ build_frame(initial_frame_size_in_bytes());
 293 
 294   // OSR buffer is
 295   //
 296   // locals[nlocals-1..0]
 297   // monitors[0..number_of_locks]
 298   //
 299   // locals is a direct copy of the interpreter frame so in the osr buffer
 300   // so first slot in the local array is the last local from the interpreter
 301   // and last slot is local[0] (receiver) from the interpreter
 302   //
 303   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 304   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 305   // in the interpreter frame (the method lock if a sync method)
 306 
 307   // Initialize monitors in the compiled activation.
 308   //   rcx: pointer to osr buffer
 309   //
 310   // All other registers are dead at this point and the locals will be
 311   // copied into place by code emitted in the IR.
 312 
 313   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 314   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 315     int monitor_offset = BytesPerWord * method()->max_locals() +
 316       (2 * BytesPerWord) * (number_of_locks - 1);
 317     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 318     // the OSR buffer using 2 word entries: first the lock and then
 319     // the oop.
 320     for (int i = 0; i < number_of_locks; i++) {
 321       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 322 #ifdef ASSERT
 323       // verify the interpreter's monitor has a non-null object
 324       {
 325         Label L;
 326         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD);
 327         __ jcc(Assembler::notZero, L);
 328         __ stop("locked object is NULL");
 329         __ bind(L);
 330       }
 331 #endif
 332       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 333       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 334       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 335       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 336     }
 337   }
 338 }
 339 
 340 
 341 // inline cache check; done before the frame is built.
 342 int LIR_Assembler::check_icache() {
 343   Register receiver = FrameMap::receiver_opr->as_register();
 344   Register ic_klass = IC_Klass;
 345   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
 346   const bool do_post_padding = VerifyOops || UseCompressedOops;
 347   if (!do_post_padding) {
 348     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
 349     while ((__ offset() + ic_cmp_size) % CodeEntryAlignment != 0) {
 350       __ nop();
 351     }
 352   }
 353   int offset = __ offset();
 354   __ inline_cache_check(receiver, IC_Klass);
 355   assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
 356   if (do_post_padding) {
 357     // force alignment after the cache check.
 358     // It's been verified to be aligned if !VerifyOops
 359     __ align(CodeEntryAlignment);
 360   }
 361   return offset;
 362 }
 363 
 364 
 365 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 366   jobject o = NULL;
 367   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 368   __ movoop(reg, o);
 369   patching_epilog(patch, lir_patch_normal, reg, info);
 370 }
 371 
 372 
 373 void LIR_Assembler::monitorexit(LIR_Opr obj_opr, LIR_Opr lock_opr, Register new_hdr, int monitor_no, Register exception) {
 374   if (exception->is_valid()) {
 375     // preserve exception
 376     // note: the monitor_exit runtime call is a leaf routine
 377     //       and cannot block => no GC can happen
 378     // The slow case (MonitorAccessStub) uses the first two stack slots
 379     // ([esp+0] and [esp+4]), therefore we store the exception at [esp+8]
 380     __ movptr (Address(rsp, 2*wordSize), exception);
 381   }
 382 
 383   Register obj_reg  = obj_opr->as_register();
 384   Register lock_reg = lock_opr->as_register();
 385 
 386   // setup registers (lock_reg must be rax, for lock_object)
 387   assert(obj_reg != SYNC_header && lock_reg != SYNC_header, "rax, must be available here");
 388   Register hdr = lock_reg;
 389   assert(new_hdr == SYNC_header, "wrong register");
 390   lock_reg = new_hdr;
 391   // compute pointer to BasicLock
 392   Address lock_addr = frame_map()->address_for_monitor_lock(monitor_no);
 393   __ lea(lock_reg, lock_addr);
 394   // unlock object
 395   MonitorAccessStub* slow_case = new MonitorExitStub(lock_opr, true, monitor_no);
 396   // _slow_case_stubs->append(slow_case);
 397   // temporary fix: must be created after exceptionhandler, therefore as call stub
 398   _slow_case_stubs->append(slow_case);
 399   if (UseFastLocking) {
 400     // try inlined fast unlocking first, revert to slow locking if it fails
 401     // note: lock_reg points to the displaced header since the displaced header offset is 0!
 402     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
 403     __ unlock_object(hdr, obj_reg, lock_reg, *slow_case->entry());
 404   } else {
 405     // always do slow unlocking
 406     // note: the slow unlocking code could be inlined here, however if we use
 407     //       slow unlocking, speed doesn't matter anyway and this solution is
 408     //       simpler and requires less duplicated code - additionally, the
 409     //       slow unlocking code is the same in either case which simplifies
 410     //       debugging
 411     __ jmp(*slow_case->entry());
 412   }
 413   // done
 414   __ bind(*slow_case->continuation());
 415 
 416   if (exception->is_valid()) {
 417     // restore exception
 418     __ movptr (exception, Address(rsp, 2 * wordSize));
 419   }
 420 }
 421 
 422 // This specifies the rsp decrement needed to build the frame
 423 int LIR_Assembler::initial_frame_size_in_bytes() {
 424   // if rounding, must let FrameMap know!
 425 
 426   // The frame_map records size in slots (32bit word)
 427 
 428   // subtract two words to account for return address and link
 429   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 430 }
 431 
 432 
 433 int LIR_Assembler::emit_exception_handler() {
 434   // if the last instruction is a call (typically to do a throw which
 435   // is coming at the end after block reordering) the return address
 436   // must still point into the code area in order to avoid assertion
 437   // failures when searching for the corresponding bci => add a nop
 438   // (was bug 5/14/1999 - gri)
 439   __ nop();
 440 
 441   // generate code for exception handler
 442   address handler_base = __ start_a_stub(exception_handler_size);
 443   if (handler_base == NULL) {
 444     // not enough space left for the handler
 445     bailout("exception handler overflow");
 446     return -1;
 447   }
 448 
 449   int offset = code_offset();
 450 
 451   // the exception oop and pc are in rax, and rdx
 452   // no other registers need to be preserved, so invalidate them
 453   __ invalidate_registers(false, true, true, false, true, true);
 454 
 455   // check that there is really an exception
 456   __ verify_not_null_oop(rax);
 457 
 458   // search an exception handler (rax: exception oop, rdx: throwing pc)
 459   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_nofpu_id)));
 460 
 461   __ stop("should not reach here");
 462 
 463   assert(code_offset() - offset <= exception_handler_size, "overflow");
 464   __ end_a_stub();
 465 
 466   return offset;
 467 }
 468 
 469 
 470 // Emit the code to remove the frame from the stack in the exception
 471 // unwind path.
 472 int LIR_Assembler::emit_unwind_handler() {
 473 #ifndef PRODUCT
 474   if (CommentedAssembly) {
 475     _masm->block_comment("Unwind handler");
 476   }
 477 #endif
 478 
 479   int offset = code_offset();
 480 
 481   // Fetch the exception from TLS and clear out exception related thread state
 482   __ get_thread(rsi);
 483   __ movptr(rax, Address(rsi, JavaThread::exception_oop_offset()));
 484   __ movptr(Address(rsi, JavaThread::exception_oop_offset()), (int32_t)NULL_WORD);
 485   __ movptr(Address(rsi, JavaThread::exception_pc_offset()), (int32_t)NULL_WORD);
 486 
 487   __ bind(_unwind_handler_entry);
 488   __ verify_not_null_oop(rax);
 489   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 490     __ mov(rsi, rax);  // Preserve the exception
 491   }
 492 
 493   // Preform needed unlocking
 494   MonitorExitStub* stub = NULL;
 495   if (method()->is_synchronized()) {
 496     monitor_address(0, FrameMap::rax_opr);
 497     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 498     __ unlock_object(rdi, rbx, rax, *stub->entry());
 499     __ bind(*stub->continuation());
 500   }
 501 
 502   if (compilation()->env()->dtrace_method_probes()) {
 503     __ get_thread(rax);
 504     __ movptr(Address(rsp, 0), rax);
 505     __ movoop(Address(rsp, sizeof(void*)), method()->constant_encoding());
 506     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 507   }
 508 
 509   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 510     __ mov(rax, rsi);  // Restore the exception
 511   }
 512 
 513   // remove the activation and dispatch to the unwind handler
 514   __ remove_frame(initial_frame_size_in_bytes());
 515   __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 516 
 517   // Emit the slow path assembly
 518   if (stub != NULL) {
 519     stub->emit_code(this);
 520   }
 521 
 522   return offset;
 523 }
 524 
 525 
 526 int LIR_Assembler::emit_deopt_handler() {
 527   // if the last instruction is a call (typically to do a throw which
 528   // is coming at the end after block reordering) the return address
 529   // must still point into the code area in order to avoid assertion
 530   // failures when searching for the corresponding bci => add a nop
 531   // (was bug 5/14/1999 - gri)
 532   __ nop();
 533 
 534   // generate code for exception handler
 535   address handler_base = __ start_a_stub(deopt_handler_size);
 536   if (handler_base == NULL) {
 537     // not enough space left for the handler
 538     bailout("deopt handler overflow");
 539     return -1;
 540   }
 541 
 542   int offset = code_offset();
 543   InternalAddress here(__ pc());
 544 
 545   __ pushptr(here.addr());
 546   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 547 
 548   assert(code_offset() - offset <= deopt_handler_size, "overflow");
 549   __ end_a_stub();
 550 
 551   return offset;
 552 }
 553 
 554 
 555 // This is the fast version of java.lang.String.compare; it has not
 556 // OSR-entry and therefore, we generate a slow version for OSR's
 557 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
 558   __ movptr (rbx, rcx); // receiver is in rcx
 559   __ movptr (rax, arg1->as_register());
 560 
 561   // Get addresses of first characters from both Strings
 562   __ load_heap_oop(rsi, Address(rax, java_lang_String::value_offset_in_bytes()));
 563   __ movptr       (rcx, Address(rax, java_lang_String::offset_offset_in_bytes()));
 564   __ lea          (rsi, Address(rsi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 565 
 566 
 567   // rbx, may be NULL
 568   add_debug_info_for_null_check_here(info);
 569   __ load_heap_oop(rdi, Address(rbx, java_lang_String::value_offset_in_bytes()));
 570   __ movptr       (rcx, Address(rbx, java_lang_String::offset_offset_in_bytes()));
 571   __ lea          (rdi, Address(rdi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 572 
 573   // compute minimum length (in rax) and difference of lengths (on top of stack)
 574   if (VM_Version::supports_cmov()) {
 575     __ movl     (rbx, Address(rbx, java_lang_String::count_offset_in_bytes()));
 576     __ movl     (rax, Address(rax, java_lang_String::count_offset_in_bytes()));
 577     __ mov      (rcx, rbx);
 578     __ subptr   (rbx, rax); // subtract lengths
 579     __ push     (rbx);      // result
 580     __ cmov     (Assembler::lessEqual, rax, rcx);
 581   } else {
 582     Label L;
 583     __ movl     (rbx, Address(rbx, java_lang_String::count_offset_in_bytes()));
 584     __ movl     (rcx, Address(rax, java_lang_String::count_offset_in_bytes()));
 585     __ mov      (rax, rbx);
 586     __ subptr   (rbx, rcx);
 587     __ push     (rbx);
 588     __ jcc      (Assembler::lessEqual, L);
 589     __ mov      (rax, rcx);
 590     __ bind (L);
 591   }
 592   // is minimum length 0?
 593   Label noLoop, haveResult;
 594   __ testptr (rax, rax);
 595   __ jcc (Assembler::zero, noLoop);
 596 
 597   // compare first characters
 598   __ load_unsigned_short(rcx, Address(rdi, 0));
 599   __ load_unsigned_short(rbx, Address(rsi, 0));
 600   __ subl(rcx, rbx);
 601   __ jcc(Assembler::notZero, haveResult);
 602   // starting loop
 603   __ decrement(rax); // we already tested index: skip one
 604   __ jcc(Assembler::zero, noLoop);
 605 
 606   // set rsi.edi to the end of the arrays (arrays have same length)
 607   // negate the index
 608 
 609   __ lea(rsi, Address(rsi, rax, Address::times_2, type2aelembytes(T_CHAR)));
 610   __ lea(rdi, Address(rdi, rax, Address::times_2, type2aelembytes(T_CHAR)));
 611   __ negptr(rax);
 612 
 613   // compare the strings in a loop
 614 
 615   Label loop;
 616   __ align(wordSize);
 617   __ bind(loop);
 618   __ load_unsigned_short(rcx, Address(rdi, rax, Address::times_2, 0));
 619   __ load_unsigned_short(rbx, Address(rsi, rax, Address::times_2, 0));
 620   __ subl(rcx, rbx);
 621   __ jcc(Assembler::notZero, haveResult);
 622   __ increment(rax);
 623   __ jcc(Assembler::notZero, loop);
 624 
 625   // strings are equal up to min length
 626 
 627   __ bind(noLoop);
 628   __ pop(rax);
 629   return_op(LIR_OprFact::illegalOpr);
 630 
 631   __ bind(haveResult);
 632   // leave instruction is going to discard the TOS value
 633   __ mov (rax, rcx); // result of call is in rax,
 634 }
 635 
 636 
 637 void LIR_Assembler::return_op(LIR_Opr result) {
 638   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 639   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 640     assert(result->fpu() == 0, "result must already be on TOS");
 641   }
 642 
 643   // Pop the stack before the safepoint code
 644   __ remove_frame(initial_frame_size_in_bytes());
 645 
 646   bool result_is_oop = result->is_valid() ? result->is_oop() : false;
 647 
 648   // Note: we do not need to round double result; float result has the right precision
 649   // the poll sets the condition code, but no data registers
 650   AddressLiteral polling_page(os::get_polling_page() + (SafepointPollOffset % os::vm_page_size()),
 651                               relocInfo::poll_return_type);
 652 
 653   // NOTE: the requires that the polling page be reachable else the reloc
 654   // goes to the movq that loads the address and not the faulting instruction
 655   // which breaks the signal handler code
 656 
 657   __ test32(rax, polling_page);
 658 
 659   __ ret(0);
 660 }
 661 
 662 
 663 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 664   AddressLiteral polling_page(os::get_polling_page() + (SafepointPollOffset % os::vm_page_size()),
 665                               relocInfo::poll_type);
 666 
 667   if (info != NULL) {
 668     add_debug_info_for_branch(info);
 669   } else {
 670     ShouldNotReachHere();
 671   }
 672 
 673   int offset = __ offset();
 674 
 675   // NOTE: the requires that the polling page be reachable else the reloc
 676   // goes to the movq that loads the address and not the faulting instruction
 677   // which breaks the signal handler code
 678 
 679   __ test32(rax, polling_page);
 680   return offset;
 681 }
 682 
 683 
 684 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 685   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 686 }
 687 
 688 void LIR_Assembler::swap_reg(Register a, Register b) {
 689   __ xchgptr(a, b);
 690 }
 691 
 692 
 693 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 694   assert(src->is_constant(), "should not call otherwise");
 695   assert(dest->is_register(), "should not call otherwise");
 696   LIR_Const* c = src->as_constant_ptr();
 697 
 698   switch (c->type()) {
 699     case T_INT: {
 700       assert(patch_code == lir_patch_none, "no patching handled here");
 701       __ movl(dest->as_register(), c->as_jint());
 702       break;
 703     }
 704 
 705     case T_ADDRESS: {
 706       assert(patch_code == lir_patch_none, "no patching handled here");
 707       __ movptr(dest->as_register(), c->as_jint());
 708       break;
 709     }
 710 
 711     case T_LONG: {
 712       assert(patch_code == lir_patch_none, "no patching handled here");
 713 #ifdef _LP64
 714       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 715 #else
 716       __ movptr(dest->as_register_lo(), c->as_jint_lo());
 717       __ movptr(dest->as_register_hi(), c->as_jint_hi());
 718 #endif // _LP64
 719       break;
 720     }
 721 
 722     case T_OBJECT: {
 723       if (patch_code != lir_patch_none) {
 724         jobject2reg_with_patching(dest->as_register(), info);
 725       } else {
 726         __ movoop(dest->as_register(), c->as_jobject());
 727       }
 728       break;
 729     }
 730 
 731     case T_FLOAT: {
 732       if (dest->is_single_xmm()) {
 733         if (c->is_zero_float()) {
 734           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 735         } else {
 736           __ movflt(dest->as_xmm_float_reg(),
 737                    InternalAddress(float_constant(c->as_jfloat())));
 738         }
 739       } else {
 740         assert(dest->is_single_fpu(), "must be");
 741         assert(dest->fpu_regnr() == 0, "dest must be TOS");
 742         if (c->is_zero_float()) {
 743           __ fldz();
 744         } else if (c->is_one_float()) {
 745           __ fld1();
 746         } else {
 747           __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
 748         }
 749       }
 750       break;
 751     }
 752 
 753     case T_DOUBLE: {
 754       if (dest->is_double_xmm()) {
 755         if (c->is_zero_double()) {
 756           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 757         } else {
 758           __ movdbl(dest->as_xmm_double_reg(),
 759                     InternalAddress(double_constant(c->as_jdouble())));
 760         }
 761       } else {
 762         assert(dest->is_double_fpu(), "must be");
 763         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
 764         if (c->is_zero_double()) {
 765           __ fldz();
 766         } else if (c->is_one_double()) {
 767           __ fld1();
 768         } else {
 769           __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
 770         }
 771       }
 772       break;
 773     }
 774 
 775     default:
 776       ShouldNotReachHere();
 777   }
 778 }
 779 
 780 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 781   assert(src->is_constant(), "should not call otherwise");
 782   assert(dest->is_stack(), "should not call otherwise");
 783   LIR_Const* c = src->as_constant_ptr();
 784 
 785   switch (c->type()) {
 786     case T_INT:  // fall through
 787     case T_FLOAT:
 788       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 789       break;
 790 
 791     case T_ADDRESS:
 792       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 793       break;
 794 
 795     case T_OBJECT:
 796       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject());
 797       break;
 798 
 799     case T_LONG:  // fall through
 800     case T_DOUBLE:
 801 #ifdef _LP64
 802       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 803                                             lo_word_offset_in_bytes), (intptr_t)c->as_jlong_bits());
 804 #else
 805       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 806                                               lo_word_offset_in_bytes), c->as_jint_lo_bits());
 807       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 808                                               hi_word_offset_in_bytes), c->as_jint_hi_bits());
 809 #endif // _LP64
 810       break;
 811 
 812     default:
 813       ShouldNotReachHere();
 814   }
 815 }
 816 
 817 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 818   assert(src->is_constant(), "should not call otherwise");
 819   assert(dest->is_address(), "should not call otherwise");
 820   LIR_Const* c = src->as_constant_ptr();
 821   LIR_Address* addr = dest->as_address_ptr();
 822 
 823   int null_check_here = code_offset();
 824   switch (type) {
 825     case T_INT:    // fall through
 826     case T_FLOAT:
 827       __ movl(as_Address(addr), c->as_jint_bits());
 828       break;
 829 
 830     case T_ADDRESS:
 831       __ movptr(as_Address(addr), c->as_jint_bits());
 832       break;
 833 
 834     case T_OBJECT:  // fall through
 835     case T_ARRAY:
 836       if (c->as_jobject() == NULL) {
 837         if (UseCompressedOops && !wide) {
 838           __ movl(as_Address(addr), (int32_t)NULL_WORD);
 839         } else {
 840           __ movptr(as_Address(addr), NULL_WORD);
 841         }
 842       } else {
 843         if (is_literal_address(addr)) {
 844           ShouldNotReachHere();
 845           __ movoop(as_Address(addr, noreg), c->as_jobject());
 846         } else {
 847 #ifdef _LP64
 848           __ movoop(rscratch1, c->as_jobject());
 849           if (UseCompressedOops && !wide) {
 850             __ encode_heap_oop(rscratch1);
 851             null_check_here = code_offset();
 852             __ movl(as_Address_lo(addr), rscratch1);
 853           } else {
 854             null_check_here = code_offset();
 855             __ movptr(as_Address_lo(addr), rscratch1);
 856           }
 857 #else
 858           __ movoop(as_Address(addr), c->as_jobject());
 859 #endif
 860         }
 861       }
 862       break;
 863 
 864     case T_LONG:    // fall through
 865     case T_DOUBLE:
 866 #ifdef _LP64
 867       if (is_literal_address(addr)) {
 868         ShouldNotReachHere();
 869         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 870       } else {
 871         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 872         null_check_here = code_offset();
 873         __ movptr(as_Address_lo(addr), r10);
 874       }
 875 #else
 876       // Always reachable in 32bit so this doesn't produce useless move literal
 877       __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
 878       __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
 879 #endif // _LP64
 880       break;
 881 
 882     case T_BOOLEAN: // fall through
 883     case T_BYTE:
 884       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 885       break;
 886 
 887     case T_CHAR:    // fall through
 888     case T_SHORT:
 889       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 890       break;
 891 
 892     default:
 893       ShouldNotReachHere();
 894   };
 895 
 896   if (info != NULL) {
 897     add_debug_info_for_null_check(null_check_here, info);
 898   }
 899 }
 900 
 901 
 902 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 903   assert(src->is_register(), "should not call otherwise");
 904   assert(dest->is_register(), "should not call otherwise");
 905 
 906   // move between cpu-registers
 907   if (dest->is_single_cpu()) {
 908 #ifdef _LP64
 909     if (src->type() == T_LONG) {
 910       // Can do LONG -> OBJECT
 911       move_regs(src->as_register_lo(), dest->as_register());
 912       return;
 913     }
 914 #endif
 915     assert(src->is_single_cpu(), "must match");
 916     if (src->type() == T_OBJECT) {
 917       __ verify_oop(src->as_register());
 918     }
 919     move_regs(src->as_register(), dest->as_register());
 920 
 921   } else if (dest->is_double_cpu()) {
 922 #ifdef _LP64
 923     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 924       // Surprising to me but we can see move of a long to t_object
 925       __ verify_oop(src->as_register());
 926       move_regs(src->as_register(), dest->as_register_lo());
 927       return;
 928     }
 929 #endif
 930     assert(src->is_double_cpu(), "must match");
 931     Register f_lo = src->as_register_lo();
 932     Register f_hi = src->as_register_hi();
 933     Register t_lo = dest->as_register_lo();
 934     Register t_hi = dest->as_register_hi();
 935 #ifdef _LP64
 936     assert(f_hi == f_lo, "must be same");
 937     assert(t_hi == t_lo, "must be same");
 938     move_regs(f_lo, t_lo);
 939 #else
 940     assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
 941 
 942 
 943     if (f_lo == t_hi && f_hi == t_lo) {
 944       swap_reg(f_lo, f_hi);
 945     } else if (f_hi == t_lo) {
 946       assert(f_lo != t_hi, "overwriting register");
 947       move_regs(f_hi, t_hi);
 948       move_regs(f_lo, t_lo);
 949     } else {
 950       assert(f_hi != t_lo, "overwriting register");
 951       move_regs(f_lo, t_lo);
 952       move_regs(f_hi, t_hi);
 953     }
 954 #endif // LP64
 955 
 956     // special moves from fpu-register to xmm-register
 957     // necessary for method results
 958   } else if (src->is_single_xmm() && !dest->is_single_xmm()) {
 959     __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
 960     __ fld_s(Address(rsp, 0));
 961   } else if (src->is_double_xmm() && !dest->is_double_xmm()) {
 962     __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
 963     __ fld_d(Address(rsp, 0));
 964   } else if (dest->is_single_xmm() && !src->is_single_xmm()) {
 965     __ fstp_s(Address(rsp, 0));
 966     __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
 967   } else if (dest->is_double_xmm() && !src->is_double_xmm()) {
 968     __ fstp_d(Address(rsp, 0));
 969     __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
 970 
 971     // move between xmm-registers
 972   } else if (dest->is_single_xmm()) {
 973     assert(src->is_single_xmm(), "must match");
 974     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 975   } else if (dest->is_double_xmm()) {
 976     assert(src->is_double_xmm(), "must match");
 977     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 978 
 979     // move between fpu-registers (no instruction necessary because of fpu-stack)
 980   } else if (dest->is_single_fpu() || dest->is_double_fpu()) {
 981     assert(src->is_single_fpu() || src->is_double_fpu(), "must match");
 982     assert(src->fpu() == dest->fpu(), "currently should be nothing to do");
 983   } else {
 984     ShouldNotReachHere();
 985   }
 986 }
 987 
 988 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 989   assert(src->is_register(), "should not call otherwise");
 990   assert(dest->is_stack(), "should not call otherwise");
 991 
 992   if (src->is_single_cpu()) {
 993     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 994     if (type == T_OBJECT || type == T_ARRAY) {
 995       __ verify_oop(src->as_register());
 996       __ movptr (dst, src->as_register());
 997     } else {
 998       __ movl (dst, src->as_register());
 999     }
1000 
1001   } else if (src->is_double_cpu()) {
1002     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
1003     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
1004     __ movptr (dstLO, src->as_register_lo());
1005     NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
1006 
1007   } else if (src->is_single_xmm()) {
1008     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
1009     __ movflt(dst_addr, src->as_xmm_float_reg());
1010 
1011   } else if (src->is_double_xmm()) {
1012     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
1013     __ movdbl(dst_addr, src->as_xmm_double_reg());
1014 
1015   } else if (src->is_single_fpu()) {
1016     assert(src->fpu_regnr() == 0, "argument must be on TOS");
1017     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
1018     if (pop_fpu_stack)     __ fstp_s (dst_addr);
1019     else                   __ fst_s  (dst_addr);
1020 
1021   } else if (src->is_double_fpu()) {
1022     assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
1023     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
1024     if (pop_fpu_stack)     __ fstp_d (dst_addr);
1025     else                   __ fst_d  (dst_addr);
1026 
1027   } else {
1028     ShouldNotReachHere();
1029   }
1030 }
1031 
1032 
1033 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
1034   LIR_Address* to_addr = dest->as_address_ptr();
1035   PatchingStub* patch = NULL;
1036   Register compressed_src = rscratch1;
1037 
1038   if (type == T_ARRAY || type == T_OBJECT) {
1039     __ verify_oop(src->as_register());
1040 #ifdef _LP64
1041     if (UseCompressedOops && !wide) {
1042       __ movptr(compressed_src, src->as_register());
1043       __ encode_heap_oop(compressed_src);
1044     }
1045 #endif
1046   }
1047 
1048   if (patch_code != lir_patch_none) {
1049     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1050     Address toa = as_Address(to_addr);
1051     assert(toa.disp() != 0, "must have");
1052   }
1053 
1054   int null_check_here = code_offset();
1055   switch (type) {
1056     case T_FLOAT: {
1057       if (src->is_single_xmm()) {
1058         __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
1059       } else {
1060         assert(src->is_single_fpu(), "must be");
1061         assert(src->fpu_regnr() == 0, "argument must be on TOS");
1062         if (pop_fpu_stack)      __ fstp_s(as_Address(to_addr));
1063         else                    __ fst_s (as_Address(to_addr));
1064       }
1065       break;
1066     }
1067 
1068     case T_DOUBLE: {
1069       if (src->is_double_xmm()) {
1070         __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
1071       } else {
1072         assert(src->is_double_fpu(), "must be");
1073         assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
1074         if (pop_fpu_stack)      __ fstp_d(as_Address(to_addr));
1075         else                    __ fst_d (as_Address(to_addr));
1076       }
1077       break;
1078     }
1079 
1080     case T_ARRAY:   // fall through
1081     case T_OBJECT:  // fall through
1082       if (UseCompressedOops && !wide) {
1083         __ movl(as_Address(to_addr), compressed_src);
1084       } else {
1085         __ movptr(as_Address(to_addr), src->as_register());
1086       }
1087       break;
1088     case T_ADDRESS:
1089       __ movptr(as_Address(to_addr), src->as_register());
1090       break;
1091     case T_INT:
1092       __ movl(as_Address(to_addr), src->as_register());
1093       break;
1094 
1095     case T_LONG: {
1096       Register from_lo = src->as_register_lo();
1097       Register from_hi = src->as_register_hi();
1098 #ifdef _LP64
1099       __ movptr(as_Address_lo(to_addr), from_lo);
1100 #else
1101       Register base = to_addr->base()->as_register();
1102       Register index = noreg;
1103       if (to_addr->index()->is_register()) {
1104         index = to_addr->index()->as_register();
1105       }
1106       if (base == from_lo || index == from_lo) {
1107         assert(base != from_hi, "can't be");
1108         assert(index == noreg || (index != base && index != from_hi), "can't handle this");
1109         __ movl(as_Address_hi(to_addr), from_hi);
1110         if (patch != NULL) {
1111           patching_epilog(patch, lir_patch_high, base, info);
1112           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1113           patch_code = lir_patch_low;
1114         }
1115         __ movl(as_Address_lo(to_addr), from_lo);
1116       } else {
1117         assert(index == noreg || (index != base && index != from_lo), "can't handle this");
1118         __ movl(as_Address_lo(to_addr), from_lo);
1119         if (patch != NULL) {
1120           patching_epilog(patch, lir_patch_low, base, info);
1121           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1122           patch_code = lir_patch_high;
1123         }
1124         __ movl(as_Address_hi(to_addr), from_hi);
1125       }
1126 #endif // _LP64
1127       break;
1128     }
1129 
1130     case T_BYTE:    // fall through
1131     case T_BOOLEAN: {
1132       Register src_reg = src->as_register();
1133       Address dst_addr = as_Address(to_addr);
1134       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
1135       __ movb(dst_addr, src_reg);
1136       break;
1137     }
1138 
1139     case T_CHAR:    // fall through
1140     case T_SHORT:
1141       __ movw(as_Address(to_addr), src->as_register());
1142       break;
1143 
1144     default:
1145       ShouldNotReachHere();
1146   }
1147   if (info != NULL) {
1148     add_debug_info_for_null_check(null_check_here, info);
1149   }
1150 
1151   if (patch_code != lir_patch_none) {
1152     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1153   }
1154 }
1155 
1156 
1157 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1158   assert(src->is_stack(), "should not call otherwise");
1159   assert(dest->is_register(), "should not call otherwise");
1160 
1161   if (dest->is_single_cpu()) {
1162     if (type == T_ARRAY || type == T_OBJECT) {
1163       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1164       __ verify_oop(dest->as_register());
1165     } else {
1166       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1167     }
1168 
1169   } else if (dest->is_double_cpu()) {
1170     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1171     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1172     __ movptr(dest->as_register_lo(), src_addr_LO);
1173     NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1174 
1175   } else if (dest->is_single_xmm()) {
1176     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1177     __ movflt(dest->as_xmm_float_reg(), src_addr);
1178 
1179   } else if (dest->is_double_xmm()) {
1180     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1181     __ movdbl(dest->as_xmm_double_reg(), src_addr);
1182 
1183   } else if (dest->is_single_fpu()) {
1184     assert(dest->fpu_regnr() == 0, "dest must be TOS");
1185     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1186     __ fld_s(src_addr);
1187 
1188   } else if (dest->is_double_fpu()) {
1189     assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1190     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1191     __ fld_d(src_addr);
1192 
1193   } else {
1194     ShouldNotReachHere();
1195   }
1196 }
1197 
1198 
1199 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1200   if (src->is_single_stack()) {
1201     if (type == T_OBJECT || type == T_ARRAY) {
1202       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1203       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1204     } else {
1205 #ifndef _LP64
1206       __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1207       __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1208 #else
1209       //no pushl on 64bits
1210       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1211       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1212 #endif
1213     }
1214 
1215   } else if (src->is_double_stack()) {
1216 #ifdef _LP64
1217     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1218     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1219 #else
1220     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1221     // push and pop the part at src + wordSize, adding wordSize for the previous push
1222     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1223     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1224     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1225 #endif // _LP64
1226 
1227   } else {
1228     ShouldNotReachHere();
1229   }
1230 }
1231 
1232 
1233 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
1234   assert(src->is_address(), "should not call otherwise");
1235   assert(dest->is_register(), "should not call otherwise");
1236 
1237   LIR_Address* addr = src->as_address_ptr();
1238   Address from_addr = as_Address(addr);
1239 
1240   switch (type) {
1241     case T_BOOLEAN: // fall through
1242     case T_BYTE:    // fall through
1243     case T_CHAR:    // fall through
1244     case T_SHORT:
1245       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1246         // on pre P6 processors we may get partial register stalls
1247         // so blow away the value of to_rinfo before loading a
1248         // partial word into it.  Do it here so that it precedes
1249         // the potential patch point below.
1250         __ xorptr(dest->as_register(), dest->as_register());
1251       }
1252       break;
1253   }
1254 
1255   PatchingStub* patch = NULL;
1256   if (patch_code != lir_patch_none) {
1257     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1258     assert(from_addr.disp() != 0, "must have");
1259   }
1260   if (info != NULL) {
1261     add_debug_info_for_null_check_here(info);
1262   }
1263 
1264   switch (type) {
1265     case T_FLOAT: {
1266       if (dest->is_single_xmm()) {
1267         __ movflt(dest->as_xmm_float_reg(), from_addr);
1268       } else {
1269         assert(dest->is_single_fpu(), "must be");
1270         assert(dest->fpu_regnr() == 0, "dest must be TOS");
1271         __ fld_s(from_addr);
1272       }
1273       break;
1274     }
1275 
1276     case T_DOUBLE: {
1277       if (dest->is_double_xmm()) {
1278         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1279       } else {
1280         assert(dest->is_double_fpu(), "must be");
1281         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1282         __ fld_d(from_addr);
1283       }
1284       break;
1285     }
1286 
1287     case T_OBJECT:  // fall through
1288     case T_ARRAY:   // fall through
1289       if (UseCompressedOops && !wide) {
1290         __ movl(dest->as_register(), from_addr);
1291       } else {
1292         __ movptr(dest->as_register(), from_addr);
1293       }
1294       break;
1295 
1296     case T_ADDRESS:
1297       __ movptr(dest->as_register(), from_addr);
1298       break;
1299     case T_INT:
1300       __ movl(dest->as_register(), from_addr);
1301       break;
1302 
1303     case T_LONG: {
1304       Register to_lo = dest->as_register_lo();
1305       Register to_hi = dest->as_register_hi();
1306 #ifdef _LP64
1307       __ movptr(to_lo, as_Address_lo(addr));
1308 #else
1309       Register base = addr->base()->as_register();
1310       Register index = noreg;
1311       if (addr->index()->is_register()) {
1312         index = addr->index()->as_register();
1313       }
1314       if ((base == to_lo && index == to_hi) ||
1315           (base == to_hi && index == to_lo)) {
1316         // addresses with 2 registers are only formed as a result of
1317         // array access so this code will never have to deal with
1318         // patches or null checks.
1319         assert(info == NULL && patch == NULL, "must be");
1320         __ lea(to_hi, as_Address(addr));
1321         __ movl(to_lo, Address(to_hi, 0));
1322         __ movl(to_hi, Address(to_hi, BytesPerWord));
1323       } else if (base == to_lo || index == to_lo) {
1324         assert(base != to_hi, "can't be");
1325         assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1326         __ movl(to_hi, as_Address_hi(addr));
1327         if (patch != NULL) {
1328           patching_epilog(patch, lir_patch_high, base, info);
1329           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1330           patch_code = lir_patch_low;
1331         }
1332         __ movl(to_lo, as_Address_lo(addr));
1333       } else {
1334         assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1335         __ movl(to_lo, as_Address_lo(addr));
1336         if (patch != NULL) {
1337           patching_epilog(patch, lir_patch_low, base, info);
1338           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1339           patch_code = lir_patch_high;
1340         }
1341         __ movl(to_hi, as_Address_hi(addr));
1342       }
1343 #endif // _LP64
1344       break;
1345     }
1346 
1347     case T_BOOLEAN: // fall through
1348     case T_BYTE: {
1349       Register dest_reg = dest->as_register();
1350       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1351       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1352         __ movsbl(dest_reg, from_addr);
1353       } else {
1354         __ movb(dest_reg, from_addr);
1355         __ shll(dest_reg, 24);
1356         __ sarl(dest_reg, 24);
1357       }
1358       break;
1359     }
1360 
1361     case T_CHAR: {
1362       Register dest_reg = dest->as_register();
1363       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1364       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1365         __ movzwl(dest_reg, from_addr);
1366       } else {
1367         __ movw(dest_reg, from_addr);
1368       }
1369       break;
1370     }
1371 
1372     case T_SHORT: {
1373       Register dest_reg = dest->as_register();
1374       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1375         __ movswl(dest_reg, from_addr);
1376       } else {
1377         __ movw(dest_reg, from_addr);
1378         __ shll(dest_reg, 16);
1379         __ sarl(dest_reg, 16);
1380       }
1381       break;
1382     }
1383 
1384     default:
1385       ShouldNotReachHere();
1386   }
1387 
1388   if (patch != NULL) {
1389     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1390   }
1391 
1392   if (type == T_ARRAY || type == T_OBJECT) {
1393 #ifdef _LP64
1394     if (UseCompressedOops && !wide) {
1395       __ decode_heap_oop(dest->as_register());
1396     }
1397 #endif
1398     __ verify_oop(dest->as_register());
1399   }
1400 }
1401 
1402 
1403 void LIR_Assembler::prefetchr(LIR_Opr src) {
1404   LIR_Address* addr = src->as_address_ptr();
1405   Address from_addr = as_Address(addr);
1406 
1407   if (VM_Version::supports_sse()) {
1408     switch (ReadPrefetchInstr) {
1409       case 0:
1410         __ prefetchnta(from_addr); break;
1411       case 1:
1412         __ prefetcht0(from_addr); break;
1413       case 2:
1414         __ prefetcht2(from_addr); break;
1415       default:
1416         ShouldNotReachHere(); break;
1417     }
1418   } else if (VM_Version::supports_3dnow()) {
1419     __ prefetchr(from_addr);
1420   }
1421 }
1422 
1423 
1424 void LIR_Assembler::prefetchw(LIR_Opr src) {
1425   LIR_Address* addr = src->as_address_ptr();
1426   Address from_addr = as_Address(addr);
1427 
1428   if (VM_Version::supports_sse()) {
1429     switch (AllocatePrefetchInstr) {
1430       case 0:
1431         __ prefetchnta(from_addr); break;
1432       case 1:
1433         __ prefetcht0(from_addr); break;
1434       case 2:
1435         __ prefetcht2(from_addr); break;
1436       case 3:
1437         __ prefetchw(from_addr); break;
1438       default:
1439         ShouldNotReachHere(); break;
1440     }
1441   } else if (VM_Version::supports_3dnow()) {
1442     __ prefetchw(from_addr);
1443   }
1444 }
1445 
1446 
1447 NEEDS_CLEANUP; // This could be static?
1448 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1449   int elem_size = type2aelembytes(type);
1450   switch (elem_size) {
1451     case 1: return Address::times_1;
1452     case 2: return Address::times_2;
1453     case 4: return Address::times_4;
1454     case 8: return Address::times_8;
1455   }
1456   ShouldNotReachHere();
1457   return Address::no_scale;
1458 }
1459 
1460 
1461 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1462   switch (op->code()) {
1463     case lir_idiv:
1464     case lir_irem:
1465       arithmetic_idiv(op->code(),
1466                       op->in_opr1(),
1467                       op->in_opr2(),
1468                       op->in_opr3(),
1469                       op->result_opr(),
1470                       op->info());
1471       break;
1472     default:      ShouldNotReachHere(); break;
1473   }
1474 }
1475 
1476 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1477 #ifdef ASSERT
1478   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1479   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1480   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1481 #endif
1482 
1483   if (op->cond() == lir_cond_always) {
1484     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1485     __ jmp (*(op->label()));
1486   } else {
1487     Assembler::Condition acond = Assembler::zero;
1488     if (op->code() == lir_cond_float_branch) {
1489       assert(op->ublock() != NULL, "must have unordered successor");
1490       __ jcc(Assembler::parity, *(op->ublock()->label()));
1491       switch(op->cond()) {
1492         case lir_cond_equal:        acond = Assembler::equal;      break;
1493         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1494         case lir_cond_less:         acond = Assembler::below;      break;
1495         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1496         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1497         case lir_cond_greater:      acond = Assembler::above;      break;
1498         default:                         ShouldNotReachHere();
1499       }
1500     } else {
1501       switch (op->cond()) {
1502         case lir_cond_equal:        acond = Assembler::equal;       break;
1503         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1504         case lir_cond_less:         acond = Assembler::less;        break;
1505         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1506         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1507         case lir_cond_greater:      acond = Assembler::greater;     break;
1508         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1509         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1510         default:                         ShouldNotReachHere();
1511       }
1512     }
1513     __ jcc(acond,*(op->label()));
1514   }
1515 }
1516 
1517 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1518   LIR_Opr src  = op->in_opr();
1519   LIR_Opr dest = op->result_opr();
1520 
1521   switch (op->bytecode()) {
1522     case Bytecodes::_i2l:
1523 #ifdef _LP64
1524       __ movl2ptr(dest->as_register_lo(), src->as_register());
1525 #else
1526       move_regs(src->as_register(), dest->as_register_lo());
1527       move_regs(src->as_register(), dest->as_register_hi());
1528       __ sarl(dest->as_register_hi(), 31);
1529 #endif // LP64
1530       break;
1531 
1532     case Bytecodes::_l2i:
1533       move_regs(src->as_register_lo(), dest->as_register());
1534       break;
1535 
1536     case Bytecodes::_i2b:
1537       move_regs(src->as_register(), dest->as_register());
1538       __ sign_extend_byte(dest->as_register());
1539       break;
1540 
1541     case Bytecodes::_i2c:
1542       move_regs(src->as_register(), dest->as_register());
1543       __ andl(dest->as_register(), 0xFFFF);
1544       break;
1545 
1546     case Bytecodes::_i2s:
1547       move_regs(src->as_register(), dest->as_register());
1548       __ sign_extend_short(dest->as_register());
1549       break;
1550 
1551 
1552     case Bytecodes::_f2d:
1553     case Bytecodes::_d2f:
1554       if (dest->is_single_xmm()) {
1555         __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1556       } else if (dest->is_double_xmm()) {
1557         __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1558       } else {
1559         assert(src->fpu() == dest->fpu(), "register must be equal");
1560         // do nothing (float result is rounded later through spilling)
1561       }
1562       break;
1563 
1564     case Bytecodes::_i2f:
1565     case Bytecodes::_i2d:
1566       if (dest->is_single_xmm()) {
1567         __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1568       } else if (dest->is_double_xmm()) {
1569         __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1570       } else {
1571         assert(dest->fpu() == 0, "result must be on TOS");
1572         __ movl(Address(rsp, 0), src->as_register());
1573         __ fild_s(Address(rsp, 0));
1574       }
1575       break;
1576 
1577     case Bytecodes::_f2i:
1578     case Bytecodes::_d2i:
1579       if (src->is_single_xmm()) {
1580         __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1581       } else if (src->is_double_xmm()) {
1582         __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1583       } else {
1584         assert(src->fpu() == 0, "input must be on TOS");
1585         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
1586         __ fist_s(Address(rsp, 0));
1587         __ movl(dest->as_register(), Address(rsp, 0));
1588         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1589       }
1590 
1591       // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
1592       assert(op->stub() != NULL, "stub required");
1593       __ cmpl(dest->as_register(), 0x80000000);
1594       __ jcc(Assembler::equal, *op->stub()->entry());
1595       __ bind(*op->stub()->continuation());
1596       break;
1597 
1598     case Bytecodes::_l2f:
1599     case Bytecodes::_l2d:
1600       assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)");
1601       assert(dest->fpu() == 0, "result must be on TOS");
1602 
1603       __ movptr(Address(rsp, 0),            src->as_register_lo());
1604       NOT_LP64(__ movl(Address(rsp, BytesPerWord), src->as_register_hi()));
1605       __ fild_d(Address(rsp, 0));
1606       // float result is rounded later through spilling
1607       break;
1608 
1609     case Bytecodes::_f2l:
1610     case Bytecodes::_d2l:
1611       assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)");
1612       assert(src->fpu() == 0, "input must be on TOS");
1613       assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers");
1614 
1615       // instruction sequence too long to inline it here
1616       {
1617         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1618       }
1619       break;
1620 
1621     default: ShouldNotReachHere();
1622   }
1623 }
1624 
1625 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1626   if (op->init_check()) {
1627     __ cmpl(Address(op->klass()->as_register(),
1628                     instanceKlass::init_state_offset_in_bytes() + sizeof(oopDesc)),
1629             instanceKlass::fully_initialized);
1630     add_debug_info_for_null_check_here(op->stub()->info());
1631     __ jcc(Assembler::notEqual, *op->stub()->entry());
1632   }
1633   __ allocate_object(op->obj()->as_register(),
1634                      op->tmp1()->as_register(),
1635                      op->tmp2()->as_register(),
1636                      op->header_size(),
1637                      op->object_size(),
1638                      op->klass()->as_register(),
1639                      *op->stub()->entry());
1640   __ bind(*op->stub()->continuation());
1641 }
1642 
1643 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1644   if (UseSlowPath ||
1645       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1646       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1647     __ jmp(*op->stub()->entry());
1648   } else {
1649     Register len =  op->len()->as_register();
1650     Register tmp1 = op->tmp1()->as_register();
1651     Register tmp2 = op->tmp2()->as_register();
1652     Register tmp3 = op->tmp3()->as_register();
1653     if (len == tmp1) {
1654       tmp1 = tmp3;
1655     } else if (len == tmp2) {
1656       tmp2 = tmp3;
1657     } else if (len == tmp3) {
1658       // everything is ok
1659     } else {
1660       __ mov(tmp3, len);
1661     }
1662     __ allocate_array(op->obj()->as_register(),
1663                       len,
1664                       tmp1,
1665                       tmp2,
1666                       arrayOopDesc::header_size(op->type()),
1667                       array_element_size(op->type()),
1668                       op->klass()->as_register(),
1669                       *op->stub()->entry());
1670   }
1671   __ bind(*op->stub()->continuation());
1672 }
1673 
1674 void LIR_Assembler::type_profile_helper(Register mdo,
1675                                         ciMethodData *md, ciProfileData *data,
1676                                         Register recv, Label* update_done) {
1677   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1678     Label next_test;
1679     // See if the receiver is receiver[n].
1680     __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1681     __ jccb(Assembler::notEqual, next_test);
1682     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1683     __ addptr(data_addr, DataLayout::counter_increment);
1684     __ jmp(*update_done);
1685     __ bind(next_test);
1686   }
1687 
1688   // Didn't find receiver; find next empty slot and fill it in
1689   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1690     Label next_test;
1691     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1692     __ cmpptr(recv_addr, (intptr_t)NULL_WORD);
1693     __ jccb(Assembler::notEqual, next_test);
1694     __ movptr(recv_addr, recv);
1695     __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1696     __ jmp(*update_done);
1697     __ bind(next_test);
1698   }
1699 }
1700 
1701 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1702   // we always need a stub for the failure case.
1703   CodeStub* stub = op->stub();
1704   Register obj = op->object()->as_register();
1705   Register k_RInfo = op->tmp1()->as_register();
1706   Register klass_RInfo = op->tmp2()->as_register();
1707   Register dst = op->result_opr()->as_register();
1708   ciKlass* k = op->klass();
1709   Register Rtmp1 = noreg;
1710 
1711   // check if it needs to be profiled
1712   ciMethodData* md;
1713   ciProfileData* data;
1714 
1715   if (op->should_profile()) {
1716     ciMethod* method = op->profiled_method();
1717     assert(method != NULL, "Should have method");
1718     int bci = op->profiled_bci();
1719     md = method->method_data();
1720     if (md == NULL) {
1721       bailout("out of memory building methodDataOop");
1722       return;
1723     }
1724     data = md->bci_to_data(bci);
1725     assert(data != NULL,                "need data for type check");
1726     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1727   }
1728   Label profile_cast_success, profile_cast_failure;
1729   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1730   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1731 
1732   if (obj == k_RInfo) {
1733     k_RInfo = dst;
1734   } else if (obj == klass_RInfo) {
1735     klass_RInfo = dst;
1736   }
1737   if (k->is_loaded() && !UseCompressedOops) {
1738     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1739   } else {
1740     Rtmp1 = op->tmp3()->as_register();
1741     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1742   }
1743 
1744   assert_different_registers(obj, k_RInfo, klass_RInfo);
1745   if (!k->is_loaded()) {
1746     jobject2reg_with_patching(k_RInfo, op->info_for_patch());
1747   } else {
1748 #ifdef _LP64
1749     __ movoop(k_RInfo, k->constant_encoding());
1750 #endif // _LP64
1751   }
1752   assert(obj != k_RInfo, "must be different");
1753 
1754   __ cmpptr(obj, (int32_t)NULL_WORD);
1755   if (op->should_profile()) {
1756     Label not_null;
1757     __ jccb(Assembler::notEqual, not_null);
1758     // Object is null; update MDO and exit
1759     Register mdo  = klass_RInfo;
1760     __ movoop(mdo, md->constant_encoding());
1761     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
1762     int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1763     __ orl(data_addr, header_bits);
1764     __ jmp(*obj_is_null);
1765     __ bind(not_null);
1766   } else {
1767     __ jcc(Assembler::equal, *obj_is_null);
1768   }
1769   __ verify_oop(obj);
1770 
1771   if (op->fast_check()) {
1772     // get object class
1773     // not a safepoint as obj null check happens earlier
1774 #ifdef _LP64
1775     if (UseCompressedOops) {
1776       __ load_klass(Rtmp1, obj);
1777       __ cmpptr(k_RInfo, Rtmp1);
1778     } else {
1779       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1780     }
1781 #else
1782     if (k->is_loaded()) {
1783       __ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1784     } else {
1785       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1786     }
1787 #endif
1788     __ jcc(Assembler::notEqual, *failure_target);
1789     // successful cast, fall through to profile or jump
1790   } else {
1791     // get object class
1792     // not a safepoint as obj null check happens earlier
1793     __ load_klass(klass_RInfo, obj);
1794     if (k->is_loaded()) {
1795       // See if we get an immediate positive hit
1796 #ifdef _LP64
1797       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1798 #else
1799       __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1800 #endif // _LP64
1801       if (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() != k->super_check_offset()) {
1802         __ jcc(Assembler::notEqual, *failure_target);
1803         // successful cast, fall through to profile or jump
1804       } else {
1805         // See if we get an immediate positive hit
1806         __ jcc(Assembler::equal, *success_target);
1807         // check for self
1808 #ifdef _LP64
1809         __ cmpptr(klass_RInfo, k_RInfo);
1810 #else
1811         __ cmpoop(klass_RInfo, k->constant_encoding());
1812 #endif // _LP64
1813         __ jcc(Assembler::equal, *success_target);
1814 
1815         __ push(klass_RInfo);
1816 #ifdef _LP64
1817         __ push(k_RInfo);
1818 #else
1819         __ pushoop(k->constant_encoding());
1820 #endif // _LP64
1821         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1822         __ pop(klass_RInfo);
1823         __ pop(klass_RInfo);
1824         // result is a boolean
1825         __ cmpl(klass_RInfo, 0);
1826         __ jcc(Assembler::equal, *failure_target);
1827         // successful cast, fall through to profile or jump
1828       }
1829     } else {
1830       // perform the fast part of the checking logic
1831       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1832       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1833       __ push(klass_RInfo);
1834       __ push(k_RInfo);
1835       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1836       __ pop(klass_RInfo);
1837       __ pop(k_RInfo);
1838       // result is a boolean
1839       __ cmpl(k_RInfo, 0);
1840       __ jcc(Assembler::equal, *failure_target);
1841       // successful cast, fall through to profile or jump
1842     }
1843   }
1844   if (op->should_profile()) {
1845     Register mdo  = klass_RInfo, recv = k_RInfo;
1846     __ bind(profile_cast_success);
1847     __ movoop(mdo, md->constant_encoding());
1848     __ load_klass(recv, obj);
1849     Label update_done;
1850     type_profile_helper(mdo, md, data, recv, success);
1851     __ jmp(*success);
1852 
1853     __ bind(profile_cast_failure);
1854     __ movoop(mdo, md->constant_encoding());
1855     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1856     __ subptr(counter_addr, DataLayout::counter_increment);
1857     __ jmp(*failure);
1858   }
1859   __ jmp(*success);
1860 }
1861 
1862 
1863 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1864   LIR_Code code = op->code();
1865   if (code == lir_store_check) {
1866     Register value = op->object()->as_register();
1867     Register array = op->array()->as_register();
1868     Register k_RInfo = op->tmp1()->as_register();
1869     Register klass_RInfo = op->tmp2()->as_register();
1870     Register Rtmp1 = op->tmp3()->as_register();
1871 
1872     CodeStub* stub = op->stub();
1873 
1874     // check if it needs to be profiled
1875     ciMethodData* md;
1876     ciProfileData* data;
1877 
1878     if (op->should_profile()) {
1879       ciMethod* method = op->profiled_method();
1880       assert(method != NULL, "Should have method");
1881       int bci = op->profiled_bci();
1882       md = method->method_data();
1883       if (md == NULL) {
1884         bailout("out of memory building methodDataOop");
1885         return;
1886       }
1887       data = md->bci_to_data(bci);
1888       assert(data != NULL,                "need data for type check");
1889       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1890     }
1891     Label profile_cast_success, profile_cast_failure, done;
1892     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1893     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1894 
1895     __ cmpptr(value, (int32_t)NULL_WORD);
1896     if (op->should_profile()) {
1897       Label not_null;
1898       __ jccb(Assembler::notEqual, not_null);
1899       // Object is null; update MDO and exit
1900       Register mdo  = klass_RInfo;
1901       __ movoop(mdo, md->constant_encoding());
1902       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
1903       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1904       __ orl(data_addr, header_bits);
1905       __ jmp(done);
1906       __ bind(not_null);
1907     } else {
1908       __ jcc(Assembler::equal, done);
1909     }
1910 
1911     add_debug_info_for_null_check_here(op->info_for_exception());
1912     __ load_klass(k_RInfo, array);
1913     __ load_klass(klass_RInfo, value);
1914 
1915     // get instance klass (it's already uncompressed)
1916     __ movptr(k_RInfo, Address(k_RInfo, objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc)));
1917     // perform the fast part of the checking logic
1918     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1919     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1920     __ push(klass_RInfo);
1921     __ push(k_RInfo);
1922     __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1923     __ pop(klass_RInfo);
1924     __ pop(k_RInfo);
1925     // result is a boolean
1926     __ cmpl(k_RInfo, 0);
1927     __ jcc(Assembler::equal, *failure_target);
1928     // fall through to the success case
1929 
1930     if (op->should_profile()) {
1931       Register mdo  = klass_RInfo, recv = k_RInfo;
1932       __ bind(profile_cast_success);
1933       __ movoop(mdo, md->constant_encoding());
1934       __ load_klass(recv, value);
1935       Label update_done;
1936       type_profile_helper(mdo, md, data, recv, &done);
1937       __ jmpb(done);
1938 
1939       __ bind(profile_cast_failure);
1940       __ movoop(mdo, md->constant_encoding());
1941       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1942       __ subptr(counter_addr, DataLayout::counter_increment);
1943       __ jmp(*stub->entry());
1944     }
1945 
1946     __ bind(done);
1947   } else
1948     if (code == lir_checkcast) {
1949       Register obj = op->object()->as_register();
1950       Register dst = op->result_opr()->as_register();
1951       Label success;
1952       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1953       __ bind(success);
1954       if (dst != obj) {
1955         __ mov(dst, obj);
1956       }
1957     } else
1958       if (code == lir_instanceof) {
1959         Register obj = op->object()->as_register();
1960         Register dst = op->result_opr()->as_register();
1961         Label success, failure, done;
1962         emit_typecheck_helper(op, &success, &failure, &failure);
1963         __ bind(failure);
1964         __ xorptr(dst, dst);
1965         __ jmpb(done);
1966         __ bind(success);
1967         __ movptr(dst, 1);
1968         __ bind(done);
1969       } else {
1970         ShouldNotReachHere();
1971       }
1972 
1973 }
1974 
1975 
1976 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1977   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
1978     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1979     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1980     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1981     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1982     Register addr = op->addr()->as_register();
1983     if (os::is_MP()) {
1984       __ lock();
1985     }
1986     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1987 
1988   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1989     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1990     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1991     Register newval = op->new_value()->as_register();
1992     Register cmpval = op->cmp_value()->as_register();
1993     assert(cmpval == rax, "wrong register");
1994     assert(newval != NULL, "new val must be register");
1995     assert(cmpval != newval, "cmp and new values must be in different registers");
1996     assert(cmpval != addr, "cmp and addr must be in different registers");
1997     assert(newval != addr, "new value and addr must be in different registers");
1998 
1999     if ( op->code() == lir_cas_obj) {
2000 #ifdef _LP64
2001       if (UseCompressedOops) {
2002         __ mov(rscratch1, cmpval);
2003         __ encode_heap_oop(cmpval);
2004         __ mov(rscratch2, newval);
2005         __ encode_heap_oop(rscratch2);
2006         if (os::is_MP()) {
2007           __ lock();
2008         }
2009         __ cmpxchgl(rscratch2, Address(addr, 0));
2010         __ mov(cmpval, rscratch1);
2011       } else 
2012 #endif
2013       {
2014         if (os::is_MP()) {
2015           __ lock();
2016         }
2017         __ cmpxchgptr(newval, Address(addr, 0));
2018       }
2019     } else {
2020       assert(op->code() == lir_cas_int, "lir_cas_int expected");
2021       if (os::is_MP()) {
2022         __ lock();
2023       }
2024       __ cmpxchgl(newval, Address(addr, 0));
2025     }
2026 #ifdef _LP64
2027   } else if (op->code() == lir_cas_long) {
2028     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2029     Register newval = op->new_value()->as_register_lo();
2030     Register cmpval = op->cmp_value()->as_register_lo();
2031     assert(cmpval == rax, "wrong register");
2032     assert(newval != NULL, "new val must be register");
2033     assert(cmpval != newval, "cmp and new values must be in different registers");
2034     assert(cmpval != addr, "cmp and addr must be in different registers");
2035     assert(newval != addr, "new value and addr must be in different registers");
2036     if (os::is_MP()) {
2037       __ lock();
2038     }
2039     __ cmpxchgq(newval, Address(addr, 0));
2040 #endif // _LP64
2041   } else {
2042     Unimplemented();
2043   }
2044 }
2045 
2046 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result) {
2047   Assembler::Condition acond, ncond;
2048   switch (condition) {
2049     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
2050     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
2051     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
2052     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
2053     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
2054     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
2055     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
2056     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
2057     default:                    ShouldNotReachHere();
2058   }
2059 
2060   if (opr1->is_cpu_register()) {
2061     reg2reg(opr1, result);
2062   } else if (opr1->is_stack()) {
2063     stack2reg(opr1, result, result->type());
2064   } else if (opr1->is_constant()) {
2065     const2reg(opr1, result, lir_patch_none, NULL);
2066   } else {
2067     ShouldNotReachHere();
2068   }
2069 
2070   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
2071     // optimized version that does not require a branch
2072     if (opr2->is_single_cpu()) {
2073       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
2074       __ cmov(ncond, result->as_register(), opr2->as_register());
2075     } else if (opr2->is_double_cpu()) {
2076       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2077       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2078       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
2079       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
2080     } else if (opr2->is_single_stack()) {
2081       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
2082     } else if (opr2->is_double_stack()) {
2083       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
2084       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
2085     } else {
2086       ShouldNotReachHere();
2087     }
2088 
2089   } else {
2090     Label skip;
2091     __ jcc (acond, skip);
2092     if (opr2->is_cpu_register()) {
2093       reg2reg(opr2, result);
2094     } else if (opr2->is_stack()) {
2095       stack2reg(opr2, result, result->type());
2096     } else if (opr2->is_constant()) {
2097       const2reg(opr2, result, lir_patch_none, NULL);
2098     } else {
2099       ShouldNotReachHere();
2100     }
2101     __ bind(skip);
2102   }
2103 }
2104 
2105 
2106 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
2107   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
2108 
2109   if (left->is_single_cpu()) {
2110     assert(left == dest, "left and dest must be equal");
2111     Register lreg = left->as_register();
2112 
2113     if (right->is_single_cpu()) {
2114       // cpu register - cpu register
2115       Register rreg = right->as_register();
2116       switch (code) {
2117         case lir_add: __ addl (lreg, rreg); break;
2118         case lir_sub: __ subl (lreg, rreg); break;
2119         case lir_mul: __ imull(lreg, rreg); break;
2120         default:      ShouldNotReachHere();
2121       }
2122 
2123     } else if (right->is_stack()) {
2124       // cpu register - stack
2125       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2126       switch (code) {
2127         case lir_add: __ addl(lreg, raddr); break;
2128         case lir_sub: __ subl(lreg, raddr); break;
2129         default:      ShouldNotReachHere();
2130       }
2131 
2132     } else if (right->is_constant()) {
2133       // cpu register - constant
2134       jint c = right->as_constant_ptr()->as_jint();
2135       switch (code) {
2136         case lir_add: {
2137           __ incrementl(lreg, c);
2138           break;
2139         }
2140         case lir_sub: {
2141           __ decrementl(lreg, c);
2142           break;
2143         }
2144         default: ShouldNotReachHere();
2145       }
2146 
2147     } else {
2148       ShouldNotReachHere();
2149     }
2150 
2151   } else if (left->is_double_cpu()) {
2152     assert(left == dest, "left and dest must be equal");
2153     Register lreg_lo = left->as_register_lo();
2154     Register lreg_hi = left->as_register_hi();
2155 
2156     if (right->is_double_cpu()) {
2157       // cpu register - cpu register
2158       Register rreg_lo = right->as_register_lo();
2159       Register rreg_hi = right->as_register_hi();
2160       NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
2161       LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
2162       switch (code) {
2163         case lir_add:
2164           __ addptr(lreg_lo, rreg_lo);
2165           NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2166           break;
2167         case lir_sub:
2168           __ subptr(lreg_lo, rreg_lo);
2169           NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2170           break;
2171         case lir_mul:
2172 #ifdef _LP64
2173           __ imulq(lreg_lo, rreg_lo);
2174 #else
2175           assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2176           __ imull(lreg_hi, rreg_lo);
2177           __ imull(rreg_hi, lreg_lo);
2178           __ addl (rreg_hi, lreg_hi);
2179           __ mull (rreg_lo);
2180           __ addl (lreg_hi, rreg_hi);
2181 #endif // _LP64
2182           break;
2183         default:
2184           ShouldNotReachHere();
2185       }
2186 
2187     } else if (right->is_constant()) {
2188       // cpu register - constant
2189 #ifdef _LP64
2190       jlong c = right->as_constant_ptr()->as_jlong_bits();
2191       __ movptr(r10, (intptr_t) c);
2192       switch (code) {
2193         case lir_add:
2194           __ addptr(lreg_lo, r10);
2195           break;
2196         case lir_sub:
2197           __ subptr(lreg_lo, r10);
2198           break;
2199         default:
2200           ShouldNotReachHere();
2201       }
2202 #else
2203       jint c_lo = right->as_constant_ptr()->as_jint_lo();
2204       jint c_hi = right->as_constant_ptr()->as_jint_hi();
2205       switch (code) {
2206         case lir_add:
2207           __ addptr(lreg_lo, c_lo);
2208           __ adcl(lreg_hi, c_hi);
2209           break;
2210         case lir_sub:
2211           __ subptr(lreg_lo, c_lo);
2212           __ sbbl(lreg_hi, c_hi);
2213           break;
2214         default:
2215           ShouldNotReachHere();
2216       }
2217 #endif // _LP64
2218 
2219     } else {
2220       ShouldNotReachHere();
2221     }
2222 
2223   } else if (left->is_single_xmm()) {
2224     assert(left == dest, "left and dest must be equal");
2225     XMMRegister lreg = left->as_xmm_float_reg();
2226 
2227     if (right->is_single_xmm()) {
2228       XMMRegister rreg = right->as_xmm_float_reg();
2229       switch (code) {
2230         case lir_add: __ addss(lreg, rreg);  break;
2231         case lir_sub: __ subss(lreg, rreg);  break;
2232         case lir_mul_strictfp: // fall through
2233         case lir_mul: __ mulss(lreg, rreg);  break;
2234         case lir_div_strictfp: // fall through
2235         case lir_div: __ divss(lreg, rreg);  break;
2236         default: ShouldNotReachHere();
2237       }
2238     } else {
2239       Address raddr;
2240       if (right->is_single_stack()) {
2241         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2242       } else if (right->is_constant()) {
2243         // hack for now
2244         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2245       } else {
2246         ShouldNotReachHere();
2247       }
2248       switch (code) {
2249         case lir_add: __ addss(lreg, raddr);  break;
2250         case lir_sub: __ subss(lreg, raddr);  break;
2251         case lir_mul_strictfp: // fall through
2252         case lir_mul: __ mulss(lreg, raddr);  break;
2253         case lir_div_strictfp: // fall through
2254         case lir_div: __ divss(lreg, raddr);  break;
2255         default: ShouldNotReachHere();
2256       }
2257     }
2258 
2259   } else if (left->is_double_xmm()) {
2260     assert(left == dest, "left and dest must be equal");
2261 
2262     XMMRegister lreg = left->as_xmm_double_reg();
2263     if (right->is_double_xmm()) {
2264       XMMRegister rreg = right->as_xmm_double_reg();
2265       switch (code) {
2266         case lir_add: __ addsd(lreg, rreg);  break;
2267         case lir_sub: __ subsd(lreg, rreg);  break;
2268         case lir_mul_strictfp: // fall through
2269         case lir_mul: __ mulsd(lreg, rreg);  break;
2270         case lir_div_strictfp: // fall through
2271         case lir_div: __ divsd(lreg, rreg);  break;
2272         default: ShouldNotReachHere();
2273       }
2274     } else {
2275       Address raddr;
2276       if (right->is_double_stack()) {
2277         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2278       } else if (right->is_constant()) {
2279         // hack for now
2280         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2281       } else {
2282         ShouldNotReachHere();
2283       }
2284       switch (code) {
2285         case lir_add: __ addsd(lreg, raddr);  break;
2286         case lir_sub: __ subsd(lreg, raddr);  break;
2287         case lir_mul_strictfp: // fall through
2288         case lir_mul: __ mulsd(lreg, raddr);  break;
2289         case lir_div_strictfp: // fall through
2290         case lir_div: __ divsd(lreg, raddr);  break;
2291         default: ShouldNotReachHere();
2292       }
2293     }
2294 
2295   } else if (left->is_single_fpu()) {
2296     assert(dest->is_single_fpu(),  "fpu stack allocation required");
2297 
2298     if (right->is_single_fpu()) {
2299       arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack);
2300 
2301     } else {
2302       assert(left->fpu_regnr() == 0, "left must be on TOS");
2303       assert(dest->fpu_regnr() == 0, "dest must be on TOS");
2304 
2305       Address raddr;
2306       if (right->is_single_stack()) {
2307         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2308       } else if (right->is_constant()) {
2309         address const_addr = float_constant(right->as_jfloat());
2310         assert(const_addr != NULL, "incorrect float/double constant maintainance");
2311         // hack for now
2312         raddr = __ as_Address(InternalAddress(const_addr));
2313       } else {
2314         ShouldNotReachHere();
2315       }
2316 
2317       switch (code) {
2318         case lir_add: __ fadd_s(raddr); break;
2319         case lir_sub: __ fsub_s(raddr); break;
2320         case lir_mul_strictfp: // fall through
2321         case lir_mul: __ fmul_s(raddr); break;
2322         case lir_div_strictfp: // fall through
2323         case lir_div: __ fdiv_s(raddr); break;
2324         default:      ShouldNotReachHere();
2325       }
2326     }
2327 
2328   } else if (left->is_double_fpu()) {
2329     assert(dest->is_double_fpu(),  "fpu stack allocation required");
2330 
2331     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2332       // Double values require special handling for strictfp mul/div on x86
2333       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
2334       __ fmulp(left->fpu_regnrLo() + 1);
2335     }
2336 
2337     if (right->is_double_fpu()) {
2338       arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack);
2339 
2340     } else {
2341       assert(left->fpu_regnrLo() == 0, "left must be on TOS");
2342       assert(dest->fpu_regnrLo() == 0, "dest must be on TOS");
2343 
2344       Address raddr;
2345       if (right->is_double_stack()) {
2346         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2347       } else if (right->is_constant()) {
2348         // hack for now
2349         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2350       } else {
2351         ShouldNotReachHere();
2352       }
2353 
2354       switch (code) {
2355         case lir_add: __ fadd_d(raddr); break;
2356         case lir_sub: __ fsub_d(raddr); break;
2357         case lir_mul_strictfp: // fall through
2358         case lir_mul: __ fmul_d(raddr); break;
2359         case lir_div_strictfp: // fall through
2360         case lir_div: __ fdiv_d(raddr); break;
2361         default: ShouldNotReachHere();
2362       }
2363     }
2364 
2365     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2366       // Double values require special handling for strictfp mul/div on x86
2367       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
2368       __ fmulp(dest->fpu_regnrLo() + 1);
2369     }
2370 
2371   } else if (left->is_single_stack() || left->is_address()) {
2372     assert(left == dest, "left and dest must be equal");
2373 
2374     Address laddr;
2375     if (left->is_single_stack()) {
2376       laddr = frame_map()->address_for_slot(left->single_stack_ix());
2377     } else if (left->is_address()) {
2378       laddr = as_Address(left->as_address_ptr());
2379     } else {
2380       ShouldNotReachHere();
2381     }
2382 
2383     if (right->is_single_cpu()) {
2384       Register rreg = right->as_register();
2385       switch (code) {
2386         case lir_add: __ addl(laddr, rreg); break;
2387         case lir_sub: __ subl(laddr, rreg); break;
2388         default:      ShouldNotReachHere();
2389       }
2390     } else if (right->is_constant()) {
2391       jint c = right->as_constant_ptr()->as_jint();
2392       switch (code) {
2393         case lir_add: {
2394           __ incrementl(laddr, c);
2395           break;
2396         }
2397         case lir_sub: {
2398           __ decrementl(laddr, c);
2399           break;
2400         }
2401         default: ShouldNotReachHere();
2402       }
2403     } else {
2404       ShouldNotReachHere();
2405     }
2406 
2407   } else {
2408     ShouldNotReachHere();
2409   }
2410 }
2411 
2412 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) {
2413   assert(pop_fpu_stack  || (left_index     == dest_index || right_index     == dest_index), "invalid LIR");
2414   assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR");
2415   assert(left_index == 0 || right_index == 0, "either must be on top of stack");
2416 
2417   bool left_is_tos = (left_index == 0);
2418   bool dest_is_tos = (dest_index == 0);
2419   int non_tos_index = (left_is_tos ? right_index : left_index);
2420 
2421   switch (code) {
2422     case lir_add:
2423       if (pop_fpu_stack)       __ faddp(non_tos_index);
2424       else if (dest_is_tos)    __ fadd (non_tos_index);
2425       else                     __ fadda(non_tos_index);
2426       break;
2427 
2428     case lir_sub:
2429       if (left_is_tos) {
2430         if (pop_fpu_stack)     __ fsubrp(non_tos_index);
2431         else if (dest_is_tos)  __ fsub  (non_tos_index);
2432         else                   __ fsubra(non_tos_index);
2433       } else {
2434         if (pop_fpu_stack)     __ fsubp (non_tos_index);
2435         else if (dest_is_tos)  __ fsubr (non_tos_index);
2436         else                   __ fsuba (non_tos_index);
2437       }
2438       break;
2439 
2440     case lir_mul_strictfp: // fall through
2441     case lir_mul:
2442       if (pop_fpu_stack)       __ fmulp(non_tos_index);
2443       else if (dest_is_tos)    __ fmul (non_tos_index);
2444       else                     __ fmula(non_tos_index);
2445       break;
2446 
2447     case lir_div_strictfp: // fall through
2448     case lir_div:
2449       if (left_is_tos) {
2450         if (pop_fpu_stack)     __ fdivrp(non_tos_index);
2451         else if (dest_is_tos)  __ fdiv  (non_tos_index);
2452         else                   __ fdivra(non_tos_index);
2453       } else {
2454         if (pop_fpu_stack)     __ fdivp (non_tos_index);
2455         else if (dest_is_tos)  __ fdivr (non_tos_index);
2456         else                   __ fdiva (non_tos_index);
2457       }
2458       break;
2459 
2460     case lir_rem:
2461       assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation");
2462       __ fremr(noreg);
2463       break;
2464 
2465     default:
2466       ShouldNotReachHere();
2467   }
2468 }
2469 
2470 
2471 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
2472   if (value->is_double_xmm()) {
2473     switch(code) {
2474       case lir_abs :
2475         {
2476           if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2477             __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2478           }
2479           __ andpd(dest->as_xmm_double_reg(),
2480                     ExternalAddress((address)double_signmask_pool));
2481         }
2482         break;
2483 
2484       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2485       // all other intrinsics are not available in the SSE instruction set, so FPU is used
2486       default      : ShouldNotReachHere();
2487     }
2488 
2489   } else if (value->is_double_fpu()) {
2490     assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2491     switch(code) {
2492       case lir_log   : __ flog() ; break;
2493       case lir_log10 : __ flog10() ; break;
2494       case lir_abs   : __ fabs() ; break;
2495       case lir_sqrt  : __ fsqrt(); break;
2496       case lir_sin   :
2497         // Should consider not saving rbx, if not necessary
2498         __ trigfunc('s', op->as_Op2()->fpu_stack_size());
2499         break;
2500       case lir_cos :
2501         // Should consider not saving rbx, if not necessary
2502         assert(op->as_Op2()->fpu_stack_size() <= 6, "sin and cos need two free stack slots");
2503         __ trigfunc('c', op->as_Op2()->fpu_stack_size());
2504         break;
2505       case lir_tan :
2506         // Should consider not saving rbx, if not necessary
2507         __ trigfunc('t', op->as_Op2()->fpu_stack_size());
2508         break;
2509       default      : ShouldNotReachHere();
2510     }
2511   } else {
2512     Unimplemented();
2513   }
2514 }
2515 
2516 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2517   // assert(left->destroys_register(), "check");
2518   if (left->is_single_cpu()) {
2519     Register reg = left->as_register();
2520     if (right->is_constant()) {
2521       int val = right->as_constant_ptr()->as_jint();
2522       switch (code) {
2523         case lir_logic_and: __ andl (reg, val); break;
2524         case lir_logic_or:  __ orl  (reg, val); break;
2525         case lir_logic_xor: __ xorl (reg, val); break;
2526         default: ShouldNotReachHere();
2527       }
2528     } else if (right->is_stack()) {
2529       // added support for stack operands
2530       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2531       switch (code) {
2532         case lir_logic_and: __ andl (reg, raddr); break;
2533         case lir_logic_or:  __ orl  (reg, raddr); break;
2534         case lir_logic_xor: __ xorl (reg, raddr); break;
2535         default: ShouldNotReachHere();
2536       }
2537     } else {
2538       Register rright = right->as_register();
2539       switch (code) {
2540         case lir_logic_and: __ andptr (reg, rright); break;
2541         case lir_logic_or : __ orptr  (reg, rright); break;
2542         case lir_logic_xor: __ xorptr (reg, rright); break;
2543         default: ShouldNotReachHere();
2544       }
2545     }
2546     move_regs(reg, dst->as_register());
2547   } else {
2548     Register l_lo = left->as_register_lo();
2549     Register l_hi = left->as_register_hi();
2550     if (right->is_constant()) {
2551 #ifdef _LP64
2552       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2553       switch (code) {
2554         case lir_logic_and:
2555           __ andq(l_lo, rscratch1);
2556           break;
2557         case lir_logic_or:
2558           __ orq(l_lo, rscratch1);
2559           break;
2560         case lir_logic_xor:
2561           __ xorq(l_lo, rscratch1);
2562           break;
2563         default: ShouldNotReachHere();
2564       }
2565 #else
2566       int r_lo = right->as_constant_ptr()->as_jint_lo();
2567       int r_hi = right->as_constant_ptr()->as_jint_hi();
2568       switch (code) {
2569         case lir_logic_and:
2570           __ andl(l_lo, r_lo);
2571           __ andl(l_hi, r_hi);
2572           break;
2573         case lir_logic_or:
2574           __ orl(l_lo, r_lo);
2575           __ orl(l_hi, r_hi);
2576           break;
2577         case lir_logic_xor:
2578           __ xorl(l_lo, r_lo);
2579           __ xorl(l_hi, r_hi);
2580           break;
2581         default: ShouldNotReachHere();
2582       }
2583 #endif // _LP64
2584     } else {
2585 #ifdef _LP64
2586       Register r_lo;
2587       if (right->type() == T_OBJECT || right->type() == T_ARRAY) {
2588         r_lo = right->as_register();
2589       } else {
2590         r_lo = right->as_register_lo();
2591       }
2592 #else
2593       Register r_lo = right->as_register_lo();
2594       Register r_hi = right->as_register_hi();
2595       assert(l_lo != r_hi, "overwriting registers");
2596 #endif
2597       switch (code) {
2598         case lir_logic_and:
2599           __ andptr(l_lo, r_lo);
2600           NOT_LP64(__ andptr(l_hi, r_hi);)
2601           break;
2602         case lir_logic_or:
2603           __ orptr(l_lo, r_lo);
2604           NOT_LP64(__ orptr(l_hi, r_hi);)
2605           break;
2606         case lir_logic_xor:
2607           __ xorptr(l_lo, r_lo);
2608           NOT_LP64(__ xorptr(l_hi, r_hi);)
2609           break;
2610         default: ShouldNotReachHere();
2611       }
2612     }
2613 
2614     Register dst_lo = dst->as_register_lo();
2615     Register dst_hi = dst->as_register_hi();
2616 
2617 #ifdef _LP64
2618     move_regs(l_lo, dst_lo);
2619 #else
2620     if (dst_lo == l_hi) {
2621       assert(dst_hi != l_lo, "overwriting registers");
2622       move_regs(l_hi, dst_hi);
2623       move_regs(l_lo, dst_lo);
2624     } else {
2625       assert(dst_lo != l_hi, "overwriting registers");
2626       move_regs(l_lo, dst_lo);
2627       move_regs(l_hi, dst_hi);
2628     }
2629 #endif // _LP64
2630   }
2631 }
2632 
2633 
2634 // we assume that rax, and rdx can be overwritten
2635 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2636 
2637   assert(left->is_single_cpu(),   "left must be register");
2638   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2639   assert(result->is_single_cpu(), "result must be register");
2640 
2641   //  assert(left->destroys_register(), "check");
2642   //  assert(right->destroys_register(), "check");
2643 
2644   Register lreg = left->as_register();
2645   Register dreg = result->as_register();
2646 
2647   if (right->is_constant()) {
2648     int divisor = right->as_constant_ptr()->as_jint();
2649     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2650     if (code == lir_idiv) {
2651       assert(lreg == rax, "must be rax,");
2652       assert(temp->as_register() == rdx, "tmp register must be rdx");
2653       __ cdql(); // sign extend into rdx:rax
2654       if (divisor == 2) {
2655         __ subl(lreg, rdx);
2656       } else {
2657         __ andl(rdx, divisor - 1);
2658         __ addl(lreg, rdx);
2659       }
2660       __ sarl(lreg, log2_intptr(divisor));
2661       move_regs(lreg, dreg);
2662     } else if (code == lir_irem) {
2663       Label done;
2664       __ mov(dreg, lreg);
2665       __ andl(dreg, 0x80000000 | (divisor - 1));
2666       __ jcc(Assembler::positive, done);
2667       __ decrement(dreg);
2668       __ orl(dreg, ~(divisor - 1));
2669       __ increment(dreg);
2670       __ bind(done);
2671     } else {
2672       ShouldNotReachHere();
2673     }
2674   } else {
2675     Register rreg = right->as_register();
2676     assert(lreg == rax, "left register must be rax,");
2677     assert(rreg != rdx, "right register must not be rdx");
2678     assert(temp->as_register() == rdx, "tmp register must be rdx");
2679 
2680     move_regs(lreg, rax);
2681 
2682     int idivl_offset = __ corrected_idivl(rreg);
2683     add_debug_info_for_div0(idivl_offset, info);
2684     if (code == lir_irem) {
2685       move_regs(rdx, dreg); // result is in rdx
2686     } else {
2687       move_regs(rax, dreg);
2688     }
2689   }
2690 }
2691 
2692 
2693 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2694   if (opr1->is_single_cpu()) {
2695     Register reg1 = opr1->as_register();
2696     if (opr2->is_single_cpu()) {
2697       // cpu register - cpu register
2698       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2699         __ cmpptr(reg1, opr2->as_register());
2700       } else {
2701         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
2702         __ cmpl(reg1, opr2->as_register());
2703       }
2704     } else if (opr2->is_stack()) {
2705       // cpu register - stack
2706       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2707         __ cmpptr(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2708       } else {
2709         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2710       }
2711     } else if (opr2->is_constant()) {
2712       // cpu register - constant
2713       LIR_Const* c = opr2->as_constant_ptr();
2714       if (c->type() == T_INT) {
2715         __ cmpl(reg1, c->as_jint());
2716       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2717         // In 64bit oops are single register
2718         jobject o = c->as_jobject();
2719         if (o == NULL) {
2720           __ cmpptr(reg1, (int32_t)NULL_WORD);
2721         } else {
2722 #ifdef _LP64
2723           __ movoop(rscratch1, o);
2724           __ cmpptr(reg1, rscratch1);
2725 #else
2726           __ cmpoop(reg1, c->as_jobject());
2727 #endif // _LP64
2728         }
2729       } else {
2730         ShouldNotReachHere();
2731       }
2732       // cpu register - address
2733     } else if (opr2->is_address()) {
2734       if (op->info() != NULL) {
2735         add_debug_info_for_null_check_here(op->info());
2736       }
2737       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2738     } else {
2739       ShouldNotReachHere();
2740     }
2741 
2742   } else if(opr1->is_double_cpu()) {
2743     Register xlo = opr1->as_register_lo();
2744     Register xhi = opr1->as_register_hi();
2745     if (opr2->is_double_cpu()) {
2746 #ifdef _LP64
2747       __ cmpptr(xlo, opr2->as_register_lo());
2748 #else
2749       // cpu register - cpu register
2750       Register ylo = opr2->as_register_lo();
2751       Register yhi = opr2->as_register_hi();
2752       __ subl(xlo, ylo);
2753       __ sbbl(xhi, yhi);
2754       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2755         __ orl(xhi, xlo);
2756       }
2757 #endif // _LP64
2758     } else if (opr2->is_constant()) {
2759       // cpu register - constant 0
2760       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2761 #ifdef _LP64
2762       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2763 #else
2764       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2765       __ orl(xhi, xlo);
2766 #endif // _LP64
2767     } else {
2768       ShouldNotReachHere();
2769     }
2770 
2771   } else if (opr1->is_single_xmm()) {
2772     XMMRegister reg1 = opr1->as_xmm_float_reg();
2773     if (opr2->is_single_xmm()) {
2774       // xmm register - xmm register
2775       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2776     } else if (opr2->is_stack()) {
2777       // xmm register - stack
2778       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2779     } else if (opr2->is_constant()) {
2780       // xmm register - constant
2781       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2782     } else if (opr2->is_address()) {
2783       // xmm register - address
2784       if (op->info() != NULL) {
2785         add_debug_info_for_null_check_here(op->info());
2786       }
2787       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2788     } else {
2789       ShouldNotReachHere();
2790     }
2791 
2792   } else if (opr1->is_double_xmm()) {
2793     XMMRegister reg1 = opr1->as_xmm_double_reg();
2794     if (opr2->is_double_xmm()) {
2795       // xmm register - xmm register
2796       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2797     } else if (opr2->is_stack()) {
2798       // xmm register - stack
2799       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2800     } else if (opr2->is_constant()) {
2801       // xmm register - constant
2802       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2803     } else if (opr2->is_address()) {
2804       // xmm register - address
2805       if (op->info() != NULL) {
2806         add_debug_info_for_null_check_here(op->info());
2807       }
2808       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2809     } else {
2810       ShouldNotReachHere();
2811     }
2812 
2813   } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) {
2814     assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)");
2815     assert(opr2->is_fpu_register(), "both must be registers");
2816     __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2817 
2818   } else if (opr1->is_address() && opr2->is_constant()) {
2819     LIR_Const* c = opr2->as_constant_ptr();
2820 #ifdef _LP64
2821     if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2822       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2823       __ movoop(rscratch1, c->as_jobject());
2824     }
2825 #endif // LP64
2826     if (op->info() != NULL) {
2827       add_debug_info_for_null_check_here(op->info());
2828     }
2829     // special case: address - constant
2830     LIR_Address* addr = opr1->as_address_ptr();
2831     if (c->type() == T_INT) {
2832       __ cmpl(as_Address(addr), c->as_jint());
2833     } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
2834 #ifdef _LP64
2835       // %%% Make this explode if addr isn't reachable until we figure out a
2836       // better strategy by giving noreg as the temp for as_Address
2837       __ cmpptr(rscratch1, as_Address(addr, noreg));
2838 #else
2839       __ cmpoop(as_Address(addr), c->as_jobject());
2840 #endif // _LP64
2841     } else {
2842       ShouldNotReachHere();
2843     }
2844 
2845   } else {
2846     ShouldNotReachHere();
2847   }
2848 }
2849 
2850 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2851   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2852     if (left->is_single_xmm()) {
2853       assert(right->is_single_xmm(), "must match");
2854       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2855     } else if (left->is_double_xmm()) {
2856       assert(right->is_double_xmm(), "must match");
2857       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2858 
2859     } else {
2860       assert(left->is_single_fpu() || left->is_double_fpu(), "must be");
2861       assert(right->is_single_fpu() || right->is_double_fpu(), "must match");
2862 
2863       assert(left->fpu() == 0, "left must be on TOS");
2864       __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2865                   op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2866     }
2867   } else {
2868     assert(code == lir_cmp_l2i, "check");
2869 #ifdef _LP64
2870     Label done;
2871     Register dest = dst->as_register();
2872     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2873     __ movl(dest, -1);
2874     __ jccb(Assembler::less, done);
2875     __ set_byte_if_not_zero(dest);
2876     __ movzbl(dest, dest);
2877     __ bind(done);
2878 #else
2879     __ lcmp2int(left->as_register_hi(),
2880                 left->as_register_lo(),
2881                 right->as_register_hi(),
2882                 right->as_register_lo());
2883     move_regs(left->as_register_hi(), dst->as_register());
2884 #endif // _LP64
2885   }
2886 }
2887 
2888 
2889 void LIR_Assembler::align_call(LIR_Code code) {
2890   if (os::is_MP()) {
2891     // make sure that the displacement word of the call ends up word aligned
2892     int offset = __ offset();
2893     switch (code) {
2894       case lir_static_call:
2895       case lir_optvirtual_call:
2896       case lir_dynamic_call:
2897         offset += NativeCall::displacement_offset;
2898         break;
2899       case lir_icvirtual_call:
2900         offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
2901       break;
2902       case lir_virtual_call:  // currently, sparc-specific for niagara
2903       default: ShouldNotReachHere();
2904     }
2905     while (offset++ % BytesPerWord != 0) {
2906       __ nop();
2907     }
2908   }
2909 }
2910 
2911 
2912 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2913   assert(!os::is_MP() || (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2914          "must be aligned");
2915   __ call(AddressLiteral(op->addr(), rtype));
2916   add_call_info(code_offset(), op->info());
2917 }
2918 
2919 
2920 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2921   RelocationHolder rh = virtual_call_Relocation::spec(pc());
2922   __ movoop(IC_Klass, (jobject)Universe::non_oop_word());
2923   assert(!os::is_MP() ||
2924          (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2925          "must be aligned");
2926   __ call(AddressLiteral(op->addr(), rh));
2927   add_call_info(code_offset(), op->info());
2928 }
2929 
2930 
2931 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2932 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2933   ShouldNotReachHere();
2934 }
2935 
2936 
2937 void LIR_Assembler::emit_static_call_stub() {
2938   address call_pc = __ pc();
2939   address stub = __ start_a_stub(call_stub_size);
2940   if (stub == NULL) {
2941     bailout("static call stub overflow");
2942     return;
2943   }
2944 
2945   int start = __ offset();
2946   if (os::is_MP()) {
2947     // make sure that the displacement word of the call ends up word aligned
2948     int offset = __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset;
2949     while (offset++ % BytesPerWord != 0) {
2950       __ nop();
2951     }
2952   }
2953   __ relocate(static_stub_Relocation::spec(call_pc));
2954   __ movoop(rbx, (jobject)NULL);
2955   // must be set to -1 at code generation time
2956   assert(!os::is_MP() || ((__ offset() + 1) % BytesPerWord) == 0, "must be aligned on MP");
2957   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2958   __ jump(RuntimeAddress(__ pc()));
2959 
2960   assert(__ offset() - start <= call_stub_size, "stub too big");
2961   __ end_a_stub();
2962 }
2963 
2964 
2965 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2966   assert(exceptionOop->as_register() == rax, "must match");
2967   assert(exceptionPC->as_register() == rdx, "must match");
2968 
2969   // exception object is not added to oop map by LinearScan
2970   // (LinearScan assumes that no oops are in fixed registers)
2971   info->add_register_oop(exceptionOop);
2972   Runtime1::StubID unwind_id;
2973 
2974   // get current pc information
2975   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2976   int pc_for_athrow_offset = __ offset();
2977   InternalAddress pc_for_athrow(__ pc());
2978   __ lea(exceptionPC->as_register(), pc_for_athrow);
2979   add_call_info(pc_for_athrow_offset, info); // for exception handler
2980 
2981   __ verify_not_null_oop(rax);
2982   // search an exception handler (rax: exception oop, rdx: throwing pc)
2983   if (compilation()->has_fpu_code()) {
2984     unwind_id = Runtime1::handle_exception_id;
2985   } else {
2986     unwind_id = Runtime1::handle_exception_nofpu_id;
2987   }
2988   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2989 
2990   // enough room for two byte trap
2991   __ nop();
2992 }
2993 
2994 
2995 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2996   assert(exceptionOop->as_register() == rax, "must match");
2997 
2998   __ jmp(_unwind_handler_entry);
2999 }
3000 
3001 
3002 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
3003 
3004   // optimized version for linear scan:
3005   // * count must be already in ECX (guaranteed by LinearScan)
3006   // * left and dest must be equal
3007   // * tmp must be unused
3008   assert(count->as_register() == SHIFT_count, "count must be in ECX");
3009   assert(left == dest, "left and dest must be equal");
3010   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3011 
3012   if (left->is_single_cpu()) {
3013     Register value = left->as_register();
3014     assert(value != SHIFT_count, "left cannot be ECX");
3015 
3016     switch (code) {
3017       case lir_shl:  __ shll(value); break;
3018       case lir_shr:  __ sarl(value); break;
3019       case lir_ushr: __ shrl(value); break;
3020       default: ShouldNotReachHere();
3021     }
3022   } else if (left->is_double_cpu()) {
3023     Register lo = left->as_register_lo();
3024     Register hi = left->as_register_hi();
3025     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
3026 #ifdef _LP64
3027     switch (code) {
3028       case lir_shl:  __ shlptr(lo);        break;
3029       case lir_shr:  __ sarptr(lo);        break;
3030       case lir_ushr: __ shrptr(lo);        break;
3031       default: ShouldNotReachHere();
3032     }
3033 #else
3034 
3035     switch (code) {
3036       case lir_shl:  __ lshl(hi, lo);        break;
3037       case lir_shr:  __ lshr(hi, lo, true);  break;
3038       case lir_ushr: __ lshr(hi, lo, false); break;
3039       default: ShouldNotReachHere();
3040     }
3041 #endif // LP64
3042   } else {
3043     ShouldNotReachHere();
3044   }
3045 }
3046 
3047 
3048 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
3049   if (dest->is_single_cpu()) {
3050     // first move left into dest so that left is not destroyed by the shift
3051     Register value = dest->as_register();
3052     count = count & 0x1F; // Java spec
3053 
3054     move_regs(left->as_register(), value);
3055     switch (code) {
3056       case lir_shl:  __ shll(value, count); break;
3057       case lir_shr:  __ sarl(value, count); break;
3058       case lir_ushr: __ shrl(value, count); break;
3059       default: ShouldNotReachHere();
3060     }
3061   } else if (dest->is_double_cpu()) {
3062 #ifndef _LP64
3063     Unimplemented();
3064 #else
3065     // first move left into dest so that left is not destroyed by the shift
3066     Register value = dest->as_register_lo();
3067     count = count & 0x1F; // Java spec
3068 
3069     move_regs(left->as_register_lo(), value);
3070     switch (code) {
3071       case lir_shl:  __ shlptr(value, count); break;
3072       case lir_shr:  __ sarptr(value, count); break;
3073       case lir_ushr: __ shrptr(value, count); break;
3074       default: ShouldNotReachHere();
3075     }
3076 #endif // _LP64
3077   } else {
3078     ShouldNotReachHere();
3079   }
3080 }
3081 
3082 
3083 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
3084   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3085   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3086   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3087   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
3088 }
3089 
3090 
3091 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
3092   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3093   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3094   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3095   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3096 }
3097 
3098 
3099 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
3100   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3101   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3102   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3103   __ movoop (Address(rsp, offset_from_rsp_in_bytes), o);
3104 }
3105 
3106 
3107 // This code replaces a call to arraycopy; no exception may
3108 // be thrown in this code, they must be thrown in the System.arraycopy
3109 // activation frame; we could save some checks if this would not be the case
3110 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3111   ciArrayKlass* default_type = op->expected_type();
3112   Register src = op->src()->as_register();
3113   Register dst = op->dst()->as_register();
3114   Register src_pos = op->src_pos()->as_register();
3115   Register dst_pos = op->dst_pos()->as_register();
3116   Register length  = op->length()->as_register();
3117   Register tmp = op->tmp()->as_register();
3118 
3119   CodeStub* stub = op->stub();
3120   int flags = op->flags();
3121   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
3122   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
3123 
3124   // if we don't know anything or it's an object array, just go through the generic arraycopy
3125   if (default_type == NULL) {
3126     Label done;
3127     // save outgoing arguments on stack in case call to System.arraycopy is needed
3128     // HACK ALERT. This code used to push the parameters in a hardwired fashion
3129     // for interpreter calling conventions. Now we have to do it in new style conventions.
3130     // For the moment until C1 gets the new register allocator I just force all the
3131     // args to the right place (except the register args) and then on the back side
3132     // reload the register args properly if we go slow path. Yuck
3133 
3134     // These are proper for the calling convention
3135 
3136     store_parameter(length, 2);
3137     store_parameter(dst_pos, 1);
3138     store_parameter(dst, 0);
3139 
3140     // these are just temporary placements until we need to reload
3141     store_parameter(src_pos, 3);
3142     store_parameter(src, 4);
3143     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3144 
3145     address entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
3146 
3147     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
3148 #ifdef _LP64
3149     // The arguments are in java calling convention so we can trivially shift them to C
3150     // convention
3151     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
3152     __ mov(c_rarg0, j_rarg0);
3153     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
3154     __ mov(c_rarg1, j_rarg1);
3155     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
3156     __ mov(c_rarg2, j_rarg2);
3157     assert_different_registers(c_rarg3, j_rarg4);
3158     __ mov(c_rarg3, j_rarg3);
3159 #ifdef _WIN64
3160     // Allocate abi space for args but be sure to keep stack aligned
3161     __ subptr(rsp, 6*wordSize);
3162     store_parameter(j_rarg4, 4);
3163     __ call(RuntimeAddress(entry));
3164     __ addptr(rsp, 6*wordSize);
3165 #else
3166     __ mov(c_rarg4, j_rarg4);
3167     __ call(RuntimeAddress(entry));
3168 #endif // _WIN64
3169 #else
3170     __ push(length);
3171     __ push(dst_pos);
3172     __ push(dst);
3173     __ push(src_pos);
3174     __ push(src);
3175     __ call_VM_leaf(entry, 5); // removes pushed parameter from the stack
3176 
3177 #endif // _LP64
3178 
3179     __ cmpl(rax, 0);
3180     __ jcc(Assembler::equal, *stub->continuation());
3181 
3182     // Reload values from the stack so they are where the stub
3183     // expects them.
3184     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3185     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3186     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3187     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3188     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3189     __ jmp(*stub->entry());
3190 
3191     __ bind(*stub->continuation());
3192     return;
3193   }
3194 
3195   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3196 
3197   int elem_size = type2aelembytes(basic_type);
3198   int shift_amount;
3199   Address::ScaleFactor scale;
3200 
3201   switch (elem_size) {
3202     case 1 :
3203       shift_amount = 0;
3204       scale = Address::times_1;
3205       break;
3206     case 2 :
3207       shift_amount = 1;
3208       scale = Address::times_2;
3209       break;
3210     case 4 :
3211       shift_amount = 2;
3212       scale = Address::times_4;
3213       break;
3214     case 8 :
3215       shift_amount = 3;
3216       scale = Address::times_8;
3217       break;
3218     default:
3219       ShouldNotReachHere();
3220   }
3221 
3222   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3223   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3224   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3225   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3226 
3227   // length and pos's are all sign extended at this point on 64bit
3228 
3229   // test for NULL
3230   if (flags & LIR_OpArrayCopy::src_null_check) {
3231     __ testptr(src, src);
3232     __ jcc(Assembler::zero, *stub->entry());
3233   }
3234   if (flags & LIR_OpArrayCopy::dst_null_check) {
3235     __ testptr(dst, dst);
3236     __ jcc(Assembler::zero, *stub->entry());
3237   }
3238 
3239   // check if negative
3240   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3241     __ testl(src_pos, src_pos);
3242     __ jcc(Assembler::less, *stub->entry());
3243   }
3244   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3245     __ testl(dst_pos, dst_pos);
3246     __ jcc(Assembler::less, *stub->entry());
3247   }
3248   if (flags & LIR_OpArrayCopy::length_positive_check) {
3249     __ testl(length, length);
3250     __ jcc(Assembler::less, *stub->entry());
3251   }
3252 
3253   if (flags & LIR_OpArrayCopy::src_range_check) {
3254     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3255     __ cmpl(tmp, src_length_addr);
3256     __ jcc(Assembler::above, *stub->entry());
3257   }
3258   if (flags & LIR_OpArrayCopy::dst_range_check) {
3259     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3260     __ cmpl(tmp, dst_length_addr);
3261     __ jcc(Assembler::above, *stub->entry());
3262   }
3263 
3264   if (flags & LIR_OpArrayCopy::type_check) {
3265     if (UseCompressedOops) {
3266       __ movl(tmp, src_klass_addr);
3267       __ cmpl(tmp, dst_klass_addr);
3268     } else {
3269       __ movptr(tmp, src_klass_addr);
3270       __ cmpptr(tmp, dst_klass_addr);
3271     }
3272     __ jcc(Assembler::notEqual, *stub->entry());
3273   }
3274 
3275 #ifdef ASSERT
3276   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3277     // Sanity check the known type with the incoming class.  For the
3278     // primitive case the types must match exactly with src.klass and
3279     // dst.klass each exactly matching the default type.  For the
3280     // object array case, if no type check is needed then either the
3281     // dst type is exactly the expected type and the src type is a
3282     // subtype which we can't check or src is the same array as dst
3283     // but not necessarily exactly of type default_type.
3284     Label known_ok, halt;
3285     __ movoop(tmp, default_type->constant_encoding());
3286 #ifdef _LP64
3287     if (UseCompressedOops) {
3288       __ encode_heap_oop(tmp);
3289     }
3290 #endif
3291 
3292     if (basic_type != T_OBJECT) {
3293 
3294       if (UseCompressedOops) __ cmpl(tmp, dst_klass_addr);
3295       else                   __ cmpptr(tmp, dst_klass_addr);
3296       __ jcc(Assembler::notEqual, halt);
3297       if (UseCompressedOops) __ cmpl(tmp, src_klass_addr);
3298       else                   __ cmpptr(tmp, src_klass_addr);
3299       __ jcc(Assembler::equal, known_ok);
3300     } else {
3301       if (UseCompressedOops) __ cmpl(tmp, dst_klass_addr);
3302       else                   __ cmpptr(tmp, dst_klass_addr);
3303       __ jcc(Assembler::equal, known_ok);
3304       __ cmpptr(src, dst);
3305       __ jcc(Assembler::equal, known_ok);
3306     }
3307     __ bind(halt);
3308     __ stop("incorrect type information in arraycopy");
3309     __ bind(known_ok);
3310   }
3311 #endif
3312 
3313   if (shift_amount > 0 && basic_type != T_OBJECT) {
3314     __ shlptr(length, shift_amount);
3315   }
3316 
3317 #ifdef _LP64
3318   assert_different_registers(c_rarg0, dst, dst_pos, length);
3319   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3320   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3321   assert_different_registers(c_rarg1, length);
3322   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3323   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3324   __ mov(c_rarg2, length);
3325 
3326 #else
3327   __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3328   store_parameter(tmp, 0);
3329   __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3330   store_parameter(tmp, 1);
3331   store_parameter(length, 2);
3332 #endif // _LP64
3333   if (basic_type == T_OBJECT) {
3334     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Runtime1::oop_arraycopy), 0);
3335   } else {
3336     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Runtime1::primitive_arraycopy), 0);
3337   }
3338 
3339   __ bind(*stub->continuation());
3340 }
3341 
3342 
3343 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3344   Register obj = op->obj_opr()->as_register();  // may not be an oop
3345   Register hdr = op->hdr_opr()->as_register();
3346   Register lock = op->lock_opr()->as_register();
3347   if (!UseFastLocking) {
3348     __ jmp(*op->stub()->entry());
3349   } else if (op->code() == lir_lock) {
3350     Register scratch = noreg;
3351     if (UseBiasedLocking) {
3352       scratch = op->scratch_opr()->as_register();
3353     }
3354     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3355     // add debug info for NullPointerException only if one is possible
3356     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
3357     if (op->info() != NULL) {
3358       add_debug_info_for_null_check(null_check_offset, op->info());
3359     }
3360     // done
3361   } else if (op->code() == lir_unlock) {
3362     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3363     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3364   } else {
3365     Unimplemented();
3366   }
3367   __ bind(*op->stub()->continuation());
3368 }
3369 
3370 
3371 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3372   ciMethod* method = op->profiled_method();
3373   int bci          = op->profiled_bci();
3374 
3375   // Update counter for all call types
3376   ciMethodData* md = method->method_data();
3377   if (md == NULL) {
3378     bailout("out of memory building methodDataOop");
3379     return;
3380   }
3381   ciProfileData* data = md->bci_to_data(bci);
3382   assert(data->is_CounterData(), "need CounterData for calls");
3383   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3384   Register mdo  = op->mdo()->as_register();
3385   __ movoop(mdo, md->constant_encoding());
3386   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3387   Bytecodes::Code bc = method->java_code_at_bci(bci);
3388   // Perform additional virtual call profiling for invokevirtual and
3389   // invokeinterface bytecodes
3390   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
3391       C1ProfileVirtualCalls) {
3392     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3393     Register recv = op->recv()->as_register();
3394     assert_different_registers(mdo, recv);
3395     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3396     ciKlass* known_klass = op->known_holder();
3397     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3398       // We know the type that will be seen at this call site; we can
3399       // statically update the methodDataOop rather than needing to do
3400       // dynamic tests on the receiver type
3401 
3402       // NOTE: we should probably put a lock around this search to
3403       // avoid collisions by concurrent compilations
3404       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3405       uint i;
3406       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3407         ciKlass* receiver = vc_data->receiver(i);
3408         if (known_klass->equals(receiver)) {
3409           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3410           __ addptr(data_addr, DataLayout::counter_increment);
3411           return;
3412         }
3413       }
3414 
3415       // Receiver type not found in profile data; select an empty slot
3416 
3417       // Note that this is less efficient than it should be because it
3418       // always does a write to the receiver part of the
3419       // VirtualCallData rather than just the first time
3420       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3421         ciKlass* receiver = vc_data->receiver(i);
3422         if (receiver == NULL) {
3423           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3424           __ movoop(recv_addr, known_klass->constant_encoding());
3425           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3426           __ addptr(data_addr, DataLayout::counter_increment);
3427           return;
3428         }
3429       }
3430     } else {
3431       __ load_klass(recv, recv);
3432       Label update_done;
3433       type_profile_helper(mdo, md, data, recv, &update_done);
3434       // Receiver did not match any saved receiver and there is no empty row for it.
3435       // Increment total counter to indicate polymorphic case.
3436       __ addptr(counter_addr, DataLayout::counter_increment);
3437 
3438       __ bind(update_done);
3439     }
3440   } else {
3441     // Static call
3442     __ addptr(counter_addr, DataLayout::counter_increment);
3443   }
3444 }
3445 
3446 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3447   Unimplemented();
3448 }
3449 
3450 
3451 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3452   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3453 }
3454 
3455 
3456 void LIR_Assembler::align_backward_branch_target() {
3457   __ align(BytesPerWord);
3458 }
3459 
3460 
3461 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
3462   if (left->is_single_cpu()) {
3463     __ negl(left->as_register());
3464     move_regs(left->as_register(), dest->as_register());
3465 
3466   } else if (left->is_double_cpu()) {
3467     Register lo = left->as_register_lo();
3468 #ifdef _LP64
3469     Register dst = dest->as_register_lo();
3470     __ movptr(dst, lo);
3471     __ negptr(dst);
3472 #else
3473     Register hi = left->as_register_hi();
3474     __ lneg(hi, lo);
3475     if (dest->as_register_lo() == hi) {
3476       assert(dest->as_register_hi() != lo, "destroying register");
3477       move_regs(hi, dest->as_register_hi());
3478       move_regs(lo, dest->as_register_lo());
3479     } else {
3480       move_regs(lo, dest->as_register_lo());
3481       move_regs(hi, dest->as_register_hi());
3482     }
3483 #endif // _LP64
3484 
3485   } else if (dest->is_single_xmm()) {
3486     if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3487       __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3488     }
3489     __ xorps(dest->as_xmm_float_reg(),
3490              ExternalAddress((address)float_signflip_pool));
3491 
3492   } else if (dest->is_double_xmm()) {
3493     if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3494       __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3495     }
3496     __ xorpd(dest->as_xmm_double_reg(),
3497              ExternalAddress((address)double_signflip_pool));
3498 
3499   } else if (left->is_single_fpu() || left->is_double_fpu()) {
3500     assert(left->fpu() == 0, "arg must be on TOS");
3501     assert(dest->fpu() == 0, "dest must be TOS");
3502     __ fchs();
3503 
3504   } else {
3505     ShouldNotReachHere();
3506   }
3507 }
3508 
3509 
3510 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
3511   assert(addr->is_address() && dest->is_register(), "check");
3512   Register reg;
3513   reg = dest->as_pointer_register();
3514   __ lea(reg, as_Address(addr->as_address_ptr()));
3515 }
3516 
3517 
3518 
3519 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3520   assert(!tmp->is_valid(), "don't need temporary");
3521   __ call(RuntimeAddress(dest));
3522   if (info != NULL) {
3523     add_call_info_here(info);
3524   }
3525 }
3526 
3527 
3528 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3529   assert(type == T_LONG, "only for volatile long fields");
3530 
3531   if (info != NULL) {
3532     add_debug_info_for_null_check_here(info);
3533   }
3534 
3535   if (src->is_double_xmm()) {
3536     if (dest->is_double_cpu()) {
3537 #ifdef _LP64
3538       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3539 #else
3540       __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
3541       __ psrlq(src->as_xmm_double_reg(), 32);
3542       __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
3543 #endif // _LP64
3544     } else if (dest->is_double_stack()) {
3545       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3546     } else if (dest->is_address()) {
3547       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3548     } else {
3549       ShouldNotReachHere();
3550     }
3551 
3552   } else if (dest->is_double_xmm()) {
3553     if (src->is_double_stack()) {
3554       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3555     } else if (src->is_address()) {
3556       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3557     } else {
3558       ShouldNotReachHere();
3559     }
3560 
3561   } else if (src->is_double_fpu()) {
3562     assert(src->fpu_regnrLo() == 0, "must be TOS");
3563     if (dest->is_double_stack()) {
3564       __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
3565     } else if (dest->is_address()) {
3566       __ fistp_d(as_Address(dest->as_address_ptr()));
3567     } else {
3568       ShouldNotReachHere();
3569     }
3570 
3571   } else if (dest->is_double_fpu()) {
3572     assert(dest->fpu_regnrLo() == 0, "must be TOS");
3573     if (src->is_double_stack()) {
3574       __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
3575     } else if (src->is_address()) {
3576       __ fild_d(as_Address(src->as_address_ptr()));
3577     } else {
3578       ShouldNotReachHere();
3579     }
3580   } else {
3581     ShouldNotReachHere();
3582   }
3583 }
3584 
3585 
3586 void LIR_Assembler::membar() {
3587   // QQQ sparc TSO uses this,
3588   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3589 }
3590 
3591 void LIR_Assembler::membar_acquire() {
3592   // No x86 machines currently require load fences
3593   // __ load_fence();
3594 }
3595 
3596 void LIR_Assembler::membar_release() {
3597   // No x86 machines currently require store fences
3598   // __ store_fence();
3599 }
3600 
3601 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3602   assert(result_reg->is_register(), "check");
3603 #ifdef _LP64
3604   // __ get_thread(result_reg->as_register_lo());
3605   __ mov(result_reg->as_register(), r15_thread);
3606 #else
3607   __ get_thread(result_reg->as_register());
3608 #endif // _LP64
3609 }
3610 
3611 
3612 void LIR_Assembler::peephole(LIR_List*) {
3613   // do nothing for now
3614 }
3615 
3616 
3617 #undef __