1 /*
   2  * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInstance.hpp"
  35 #include "ci/ciValueKlass.hpp"
  36 #include "gc/shared/barrierSet.hpp"
  37 #include "gc/shared/cardTableBarrierSet.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "nativeInst_x86.hpp"
  40 #include "oops/oop.inline.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "vmreg_x86.inline.hpp"
  46 
  47 
  48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  49 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  50 // fast versions of NegF/NegD and AbsF/AbsD.
  51 
  52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  54   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  55   // of 128-bits operands for SSE instructions.
  56   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  57   // Store the value to a 128-bits operand.
  58   operand[0] = lo;
  59   operand[1] = hi;
  60   return operand;
  61 }
  62 
  63 // Buffer for 128-bits masks used by SSE instructions.
  64 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  65 
  66 // Static initialization during VM startup.
  67 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));
  68 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));
  69 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
  70 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
  71 
  72 
  73 NEEDS_CLEANUP // remove this definitions ?
  74 const Register IC_Klass    = rax;   // where the IC klass is cached
  75 const Register SYNC_header = rax;   // synchronization header
  76 const Register SHIFT_count = rcx;   // where count for shift operations must be
  77 
  78 #define __ _masm->
  79 
  80 
  81 static void select_different_registers(Register preserve,
  82                                        Register extra,
  83                                        Register &tmp1,
  84                                        Register &tmp2) {
  85   if (tmp1 == preserve) {
  86     assert_different_registers(tmp1, tmp2, extra);
  87     tmp1 = extra;
  88   } else if (tmp2 == preserve) {
  89     assert_different_registers(tmp1, tmp2, extra);
  90     tmp2 = extra;
  91   }
  92   assert_different_registers(preserve, tmp1, tmp2);
  93 }
  94 
  95 
  96 
  97 static void select_different_registers(Register preserve,
  98                                        Register extra,
  99                                        Register &tmp1,
 100                                        Register &tmp2,
 101                                        Register &tmp3) {
 102   if (tmp1 == preserve) {
 103     assert_different_registers(tmp1, tmp2, tmp3, extra);
 104     tmp1 = extra;
 105   } else if (tmp2 == preserve) {
 106     assert_different_registers(tmp1, tmp2, tmp3, extra);
 107     tmp2 = extra;
 108   } else if (tmp3 == preserve) {
 109     assert_different_registers(tmp1, tmp2, tmp3, extra);
 110     tmp3 = extra;
 111   }
 112   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 113 }
 114 
 115 
 116 
 117 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 118   if (opr->is_constant()) {
 119     LIR_Const* constant = opr->as_constant_ptr();
 120     switch (constant->type()) {
 121       case T_INT: {
 122         return true;
 123       }
 124 
 125       default:
 126         return false;
 127     }
 128   }
 129   return false;
 130 }
 131 
 132 
 133 LIR_Opr LIR_Assembler::receiverOpr() {
 134   return FrameMap::receiver_opr;
 135 }
 136 
 137 LIR_Opr LIR_Assembler::osrBufferPointer() {
 138   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 139 }
 140 
 141 //--------------fpu register translations-----------------------
 142 
 143 
 144 address LIR_Assembler::float_constant(float f) {
 145   address const_addr = __ float_constant(f);
 146   if (const_addr == NULL) {
 147     bailout("const section overflow");
 148     return __ code()->consts()->start();
 149   } else {
 150     return const_addr;
 151   }
 152 }
 153 
 154 
 155 address LIR_Assembler::double_constant(double d) {
 156   address const_addr = __ double_constant(d);
 157   if (const_addr == NULL) {
 158     bailout("const section overflow");
 159     return __ code()->consts()->start();
 160   } else {
 161     return const_addr;
 162   }
 163 }
 164 
 165 
 166 void LIR_Assembler::set_24bit_FPU() {
 167   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
 168 }
 169 
 170 void LIR_Assembler::reset_FPU() {
 171   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 172 }
 173 
 174 void LIR_Assembler::fpop() {
 175   __ fpop();
 176 }
 177 
 178 void LIR_Assembler::fxch(int i) {
 179   __ fxch(i);
 180 }
 181 
 182 void LIR_Assembler::fld(int i) {
 183   __ fld_s(i);
 184 }
 185 
 186 void LIR_Assembler::ffree(int i) {
 187   __ ffree(i);
 188 }
 189 
 190 void LIR_Assembler::breakpoint() {
 191   __ int3();
 192 }
 193 
 194 void LIR_Assembler::push(LIR_Opr opr) {
 195   if (opr->is_single_cpu()) {
 196     __ push_reg(opr->as_register());
 197   } else if (opr->is_double_cpu()) {
 198     NOT_LP64(__ push_reg(opr->as_register_hi()));
 199     __ push_reg(opr->as_register_lo());
 200   } else if (opr->is_stack()) {
 201     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 202   } else if (opr->is_constant()) {
 203     LIR_Const* const_opr = opr->as_constant_ptr();
 204     if (const_opr->type() == T_OBJECT || const_opr->type() == T_VALUETYPE) {
 205       __ push_oop(const_opr->as_jobject());
 206     } else if (const_opr->type() == T_INT) {
 207       __ push_jint(const_opr->as_jint());
 208     } else {
 209       ShouldNotReachHere();
 210     }
 211 
 212   } else {
 213     ShouldNotReachHere();
 214   }
 215 }
 216 
 217 void LIR_Assembler::pop(LIR_Opr opr) {
 218   if (opr->is_single_cpu()) {
 219     __ pop_reg(opr->as_register());
 220   } else {
 221     ShouldNotReachHere();
 222   }
 223 }
 224 
 225 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 226   return addr->base()->is_illegal() && addr->index()->is_illegal();
 227 }
 228 
 229 //-------------------------------------------
 230 
 231 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 232   return as_Address(addr, rscratch1);
 233 }
 234 
 235 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 236   if (addr->base()->is_illegal()) {
 237     assert(addr->index()->is_illegal(), "must be illegal too");
 238     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 239     if (! __ reachable(laddr)) {
 240       __ movptr(tmp, laddr.addr());
 241       Address res(tmp, 0);
 242       return res;
 243     } else {
 244       return __ as_Address(laddr);
 245     }
 246   }
 247 
 248   Register base = addr->base()->as_pointer_register();
 249 
 250   if (addr->index()->is_illegal()) {
 251     return Address( base, addr->disp());
 252   } else if (addr->index()->is_cpu_register()) {
 253     Register index = addr->index()->as_pointer_register();
 254     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 255   } else if (addr->index()->is_constant()) {
 256     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 257     assert(Assembler::is_simm32(addr_offset), "must be");
 258 
 259     return Address(base, addr_offset);
 260   } else {
 261     Unimplemented();
 262     return Address();
 263   }
 264 }
 265 
 266 
 267 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 268   Address base = as_Address(addr);
 269   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 270 }
 271 
 272 
 273 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 274   return as_Address(addr);
 275 }
 276 
 277 
 278 void LIR_Assembler::osr_entry() {
 279   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 280   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 281   ValueStack* entry_state = osr_entry->state();
 282   int number_of_locks = entry_state->locks_size();
 283 
 284   // we jump here if osr happens with the interpreter
 285   // state set up to continue at the beginning of the
 286   // loop that triggered osr - in particular, we have
 287   // the following registers setup:
 288   //
 289   // rcx: osr buffer
 290   //
 291 
 292   // build frame
 293   ciMethod* m = compilation()->method();
 294   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes(),
 295                  needs_stack_repair(), NULL);
 296 
 297   // OSR buffer is
 298   //
 299   // locals[nlocals-1..0]
 300   // monitors[0..number_of_locks]
 301   //
 302   // locals is a direct copy of the interpreter frame so in the osr buffer
 303   // so first slot in the local array is the last local from the interpreter
 304   // and last slot is local[0] (receiver) from the interpreter
 305   //
 306   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 307   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 308   // in the interpreter frame (the method lock if a sync method)
 309 
 310   // Initialize monitors in the compiled activation.
 311   //   rcx: pointer to osr buffer
 312   //
 313   // All other registers are dead at this point and the locals will be
 314   // copied into place by code emitted in the IR.
 315 
 316   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 317   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 318     int monitor_offset = BytesPerWord * method()->max_locals() +
 319       (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
 320     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 321     // the OSR buffer using 2 word entries: first the lock and then
 322     // the oop.
 323     for (int i = 0; i < number_of_locks; i++) {
 324       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 325 #ifdef ASSERT
 326       // verify the interpreter's monitor has a non-null object
 327       {
 328         Label L;
 329         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD);
 330         __ jcc(Assembler::notZero, L);
 331         __ stop("locked object is NULL");
 332         __ bind(L);
 333       }
 334 #endif
 335       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 336       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 337       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 338       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 339     }
 340   }
 341 }
 342 
 343 
 344 // inline cache check; done before the frame is built.
 345 int LIR_Assembler::check_icache() {
 346   Register receiver = FrameMap::receiver_opr->as_register();
 347   Register ic_klass = IC_Klass;
 348   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
 349   const bool do_post_padding = VerifyOops || UseCompressedClassPointers;
 350   if (!do_post_padding) {
 351     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
 352     __ align(CodeEntryAlignment, __ offset() + ic_cmp_size);
 353   }
 354   int offset = __ offset();
 355   __ inline_cache_check(receiver, IC_Klass);
 356   assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
 357   if (do_post_padding) {
 358     // force alignment after the cache check.
 359     // It's been verified to be aligned if !VerifyOops
 360     __ align(CodeEntryAlignment);
 361   }
 362   return offset;
 363 }
 364 
 365 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 366   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 367   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 368 
 369   Label L_skip_barrier;
 370   Register klass = rscratch1;
 371   Register thread = LP64_ONLY( r15_thread ) NOT_LP64( noreg );
 372   assert(thread != noreg, "x86_32 not implemented");
 373 
 374   __ mov_metadata(klass, method->holder()->constant_encoding());
 375   __ clinit_barrier(klass, thread, &L_skip_barrier /*L_fast_path*/);
 376 
 377   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 378 
 379   __ bind(L_skip_barrier);
 380 }
 381 
 382 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 383   jobject o = NULL;
 384   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 385   __ movoop(reg, o);
 386   patching_epilog(patch, lir_patch_normal, reg, info);
 387 }
 388 
 389 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 390   Metadata* o = NULL;
 391   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 392   __ mov_metadata(reg, o);
 393   patching_epilog(patch, lir_patch_normal, reg, info);
 394 }
 395 
 396 // This specifies the rsp decrement needed to build the frame
 397 int LIR_Assembler::initial_frame_size_in_bytes() const {
 398   // if rounding, must let FrameMap know!
 399 
 400   // The frame_map records size in slots (32bit word)
 401 
 402   // subtract two words to account for return address and link
 403   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 404 }
 405 
 406 
 407 int LIR_Assembler::emit_exception_handler() {
 408   // if the last instruction is a call (typically to do a throw which
 409   // is coming at the end after block reordering) the return address
 410   // must still point into the code area in order to avoid assertion
 411   // failures when searching for the corresponding bci => add a nop
 412   // (was bug 5/14/1999 - gri)
 413   __ nop();
 414 
 415   // generate code for exception handler
 416   address handler_base = __ start_a_stub(exception_handler_size());
 417   if (handler_base == NULL) {
 418     // not enough space left for the handler
 419     bailout("exception handler overflow");
 420     return -1;
 421   }
 422 
 423   int offset = code_offset();
 424 
 425   // the exception oop and pc are in rax, and rdx
 426   // no other registers need to be preserved, so invalidate them
 427   __ invalidate_registers(false, true, true, false, true, true);
 428 
 429   // check that there is really an exception
 430   __ verify_not_null_oop(rax);
 431 
 432   // search an exception handler (rax: exception oop, rdx: throwing pc)
 433   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 434   __ should_not_reach_here();
 435   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 436   __ end_a_stub();
 437 
 438   return offset;
 439 }
 440 
 441 
 442 // Emit the code to remove the frame from the stack in the exception
 443 // unwind path.
 444 int LIR_Assembler::emit_unwind_handler() {
 445 #ifndef PRODUCT
 446   if (CommentedAssembly) {
 447     _masm->block_comment("Unwind handler");
 448   }
 449 #endif
 450 
 451   int offset = code_offset();
 452 
 453   // Fetch the exception from TLS and clear out exception related thread state
 454   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 455   NOT_LP64(__ get_thread(rsi));
 456   __ movptr(rax, Address(thread, JavaThread::exception_oop_offset()));
 457   __ movptr(Address(thread, JavaThread::exception_oop_offset()), (intptr_t)NULL_WORD);
 458   __ movptr(Address(thread, JavaThread::exception_pc_offset()), (intptr_t)NULL_WORD);
 459 
 460   __ bind(_unwind_handler_entry);
 461   __ verify_not_null_oop(rax);
 462   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 463     __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)
 464   }
 465 
 466   // Preform needed unlocking
 467   MonitorExitStub* stub = NULL;
 468   if (method()->is_synchronized()) {
 469     monitor_address(0, FrameMap::rax_opr);
 470     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 471     __ unlock_object(rdi, rsi, rax, *stub->entry());
 472     __ bind(*stub->continuation());
 473   }
 474 
 475   if (compilation()->env()->dtrace_method_probes()) {
 476 #ifdef _LP64
 477     __ mov(rdi, r15_thread);
 478     __ mov_metadata(rsi, method()->constant_encoding());
 479 #else
 480     __ get_thread(rax);
 481     __ movptr(Address(rsp, 0), rax);
 482     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 483 #endif
 484     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 485   }
 486 
 487   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 488     __ mov(rax, rbx);  // Restore the exception
 489   }
 490 
 491   // remove the activation and dispatch to the unwind handler
 492   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 493   __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 494 
 495   // Emit the slow path assembly
 496   if (stub != NULL) {
 497     stub->emit_code(this);
 498   }
 499 
 500   return offset;
 501 }
 502 
 503 
 504 int LIR_Assembler::emit_deopt_handler() {
 505   // if the last instruction is a call (typically to do a throw which
 506   // is coming at the end after block reordering) the return address
 507   // must still point into the code area in order to avoid assertion
 508   // failures when searching for the corresponding bci => add a nop
 509   // (was bug 5/14/1999 - gri)
 510   __ nop();
 511 
 512   // generate code for exception handler
 513   address handler_base = __ start_a_stub(deopt_handler_size());
 514   if (handler_base == NULL) {
 515     // not enough space left for the handler
 516     bailout("deopt handler overflow");
 517     return -1;
 518   }
 519 
 520   int offset = code_offset();
 521   InternalAddress here(__ pc());
 522 
 523   __ pushptr(here.addr());
 524   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 525   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 526   __ end_a_stub();
 527 
 528   return offset;
 529 }
 530 
 531 
 532 void LIR_Assembler::return_op(LIR_Opr result) {
 533   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 534   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 535     assert(result->fpu() == 0, "result must already be on TOS");
 536   }
 537 
 538   ciMethod* method = compilation()->method();
 539   if (ValueTypeReturnedAsFields && method->signature()->returns_never_null()) {
 540     ciType* return_type = method->return_type();
 541     if (return_type->is_valuetype()) {
 542       ciValueKlass* vk = return_type->as_value_klass();
 543       if (vk->can_be_returned_as_fields()) {
 544 #ifndef _LP64
 545         Unimplemented();
 546 #else
 547         address unpack_handler = vk->unpack_handler();
 548         assert(unpack_handler != NULL, "must be");
 549         __ call(RuntimeAddress(unpack_handler));
 550         // At this point, rax points to the value object (for interpreter or C1 caller).
 551         // The fields of the object are copied into registers (for C2 caller).
 552 #endif
 553       }
 554     }
 555   }
 556 
 557   // Pop the stack before the safepoint code
 558   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 559 
 560   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 561     __ reserved_stack_check();
 562   }
 563 
 564   bool result_is_oop = result->is_valid() ? result->is_oop() : false;
 565 
 566   // Note: we do not need to round double result; float result has the right precision
 567   // the poll sets the condition code, but no data registers
 568 
 569   if (SafepointMechanism::uses_thread_local_poll()) {
 570 #ifdef _LP64
 571     const Register poll_addr = rscratch1;
 572     __ movptr(poll_addr, Address(r15_thread, Thread::polling_page_offset()));
 573 #else
 574     const Register poll_addr = rbx;
 575     assert(FrameMap::is_caller_save_register(poll_addr), "will overwrite");
 576     __ get_thread(poll_addr);
 577     __ movptr(poll_addr, Address(poll_addr, Thread::polling_page_offset()));
 578 #endif
 579     __ relocate(relocInfo::poll_return_type);
 580     __ testl(rax, Address(poll_addr, 0));
 581   } else {
 582     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 583 
 584     if (Assembler::is_polling_page_far()) {
 585       __ lea(rscratch1, polling_page);
 586       __ relocate(relocInfo::poll_return_type);
 587       __ testl(rax, Address(rscratch1, 0));
 588     } else {
 589       __ testl(rax, polling_page);
 590     }
 591   }
 592   __ ret(0);
 593 }
 594 
 595 
 596 int LIR_Assembler::store_value_type_fields_to_buf(ciValueKlass* vk) {
 597   return (__ store_value_type_fields_to_buf(vk, false));
 598 }
 599 
 600 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 601   guarantee(info != NULL, "Shouldn't be NULL");
 602   int offset = __ offset();
 603   if (SafepointMechanism::uses_thread_local_poll()) {
 604 #ifdef _LP64
 605     const Register poll_addr = rscratch1;
 606     __ movptr(poll_addr, Address(r15_thread, Thread::polling_page_offset()));
 607 #else
 608     assert(tmp->is_cpu_register(), "needed");
 609     const Register poll_addr = tmp->as_register();
 610     __ get_thread(poll_addr);
 611     __ movptr(poll_addr, Address(poll_addr, in_bytes(Thread::polling_page_offset())));
 612 #endif
 613     add_debug_info_for_branch(info);
 614     __ relocate(relocInfo::poll_type);
 615     address pre_pc = __ pc();
 616     __ testl(rax, Address(poll_addr, 0));
 617     address post_pc = __ pc();
 618     guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
 619   } else {
 620     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_type);
 621     if (Assembler::is_polling_page_far()) {
 622       __ lea(rscratch1, polling_page);
 623       offset = __ offset();
 624       add_debug_info_for_branch(info);
 625       __ relocate(relocInfo::poll_type);
 626       __ testl(rax, Address(rscratch1, 0));
 627     } else {
 628       add_debug_info_for_branch(info);
 629       __ testl(rax, polling_page);
 630     }
 631   }
 632   return offset;
 633 }
 634 
 635 
 636 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 637   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 638 }
 639 
 640 void LIR_Assembler::swap_reg(Register a, Register b) {
 641   __ xchgptr(a, b);
 642 }
 643 
 644 
 645 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 646   assert(src->is_constant(), "should not call otherwise");
 647   assert(dest->is_register(), "should not call otherwise");
 648   LIR_Const* c = src->as_constant_ptr();
 649 
 650   switch (c->type()) {
 651     case T_INT: {
 652       assert(patch_code == lir_patch_none, "no patching handled here");
 653       __ movl(dest->as_register(), c->as_jint());
 654       break;
 655     }
 656 
 657     case T_ADDRESS: {
 658       assert(patch_code == lir_patch_none, "no patching handled here");
 659       __ movptr(dest->as_register(), c->as_jint());
 660       break;
 661     }
 662 
 663     case T_LONG: {
 664       assert(patch_code == lir_patch_none, "no patching handled here");
 665 #ifdef _LP64
 666       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 667 #else
 668       __ movptr(dest->as_register_lo(), c->as_jint_lo());
 669       __ movptr(dest->as_register_hi(), c->as_jint_hi());
 670 #endif // _LP64
 671       break;
 672     }
 673 
 674     case T_VALUETYPE: // Fall through
 675     case T_OBJECT: {
 676       if (patch_code != lir_patch_none) {
 677         jobject2reg_with_patching(dest->as_register(), info);
 678       } else {
 679         __ movoop(dest->as_register(), c->as_jobject());
 680       }
 681       break;
 682     }
 683 
 684     case T_METADATA: {
 685       if (patch_code != lir_patch_none) {
 686         klass2reg_with_patching(dest->as_register(), info);
 687       } else {
 688         __ mov_metadata(dest->as_register(), c->as_metadata());
 689       }
 690       break;
 691     }
 692 
 693     case T_FLOAT: {
 694       if (dest->is_single_xmm()) {
 695         if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_float()) {
 696           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 697         } else {
 698           __ movflt(dest->as_xmm_float_reg(),
 699                    InternalAddress(float_constant(c->as_jfloat())));
 700         }
 701       } else {
 702         assert(dest->is_single_fpu(), "must be");
 703         assert(dest->fpu_regnr() == 0, "dest must be TOS");
 704         if (c->is_zero_float()) {
 705           __ fldz();
 706         } else if (c->is_one_float()) {
 707           __ fld1();
 708         } else {
 709           __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
 710         }
 711       }
 712       break;
 713     }
 714 
 715     case T_DOUBLE: {
 716       if (dest->is_double_xmm()) {
 717         if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_double()) {
 718           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 719         } else {
 720           __ movdbl(dest->as_xmm_double_reg(),
 721                     InternalAddress(double_constant(c->as_jdouble())));
 722         }
 723       } else {
 724         assert(dest->is_double_fpu(), "must be");
 725         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
 726         if (c->is_zero_double()) {
 727           __ fldz();
 728         } else if (c->is_one_double()) {
 729           __ fld1();
 730         } else {
 731           __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
 732         }
 733       }
 734       break;
 735     }
 736 
 737     default:
 738       ShouldNotReachHere();
 739   }
 740 }
 741 
 742 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 743   assert(src->is_constant(), "should not call otherwise");
 744   assert(dest->is_stack(), "should not call otherwise");
 745   LIR_Const* c = src->as_constant_ptr();
 746 
 747   switch (c->type()) {
 748     case T_INT:  // fall through
 749     case T_FLOAT:
 750       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 751       break;
 752 
 753     case T_ADDRESS:
 754       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 755       break;
 756 
 757     case T_VALUETYPE: // Fall through
 758     case T_OBJECT:
 759       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject());
 760       break;
 761 
 762     case T_LONG:  // fall through
 763     case T_DOUBLE:
 764 #ifdef _LP64
 765       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 766                                             lo_word_offset_in_bytes), (intptr_t)c->as_jlong_bits());
 767 #else
 768       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 769                                               lo_word_offset_in_bytes), c->as_jint_lo_bits());
 770       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 771                                               hi_word_offset_in_bytes), c->as_jint_hi_bits());
 772 #endif // _LP64
 773       break;
 774 
 775     default:
 776       ShouldNotReachHere();
 777   }
 778 }
 779 
 780 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 781   assert(src->is_constant(), "should not call otherwise");
 782   assert(dest->is_address(), "should not call otherwise");
 783   LIR_Const* c = src->as_constant_ptr();
 784   LIR_Address* addr = dest->as_address_ptr();
 785 
 786   int null_check_here = code_offset();
 787   switch (type) {
 788     case T_INT:    // fall through
 789     case T_FLOAT:
 790       __ movl(as_Address(addr), c->as_jint_bits());
 791       break;
 792 
 793     case T_ADDRESS:
 794       __ movptr(as_Address(addr), c->as_jint_bits());
 795       break;
 796 
 797     case T_VALUETYPE: // fall through
 798     case T_OBJECT:  // fall through
 799     case T_ARRAY:
 800       if (c->as_jobject() == NULL) {
 801         if (UseCompressedOops && !wide) {
 802           __ movl(as_Address(addr), (int32_t)NULL_WORD);
 803         } else {
 804 #ifdef _LP64
 805           __ xorptr(rscratch1, rscratch1);
 806           null_check_here = code_offset();
 807           __ movptr(as_Address(addr), rscratch1);
 808 #else
 809           __ movptr(as_Address(addr), NULL_WORD);
 810 #endif
 811         }
 812       } else {
 813         if (is_literal_address(addr)) {
 814           ShouldNotReachHere();
 815           __ movoop(as_Address(addr, noreg), c->as_jobject());
 816         } else {
 817 #ifdef _LP64
 818           __ movoop(rscratch1, c->as_jobject());
 819           if (UseCompressedOops && !wide) {
 820             __ encode_heap_oop(rscratch1);
 821             null_check_here = code_offset();
 822             __ movl(as_Address_lo(addr), rscratch1);
 823           } else {
 824             null_check_here = code_offset();
 825             __ movptr(as_Address_lo(addr), rscratch1);
 826           }
 827 #else
 828           __ movoop(as_Address(addr), c->as_jobject());
 829 #endif
 830         }
 831       }
 832       break;
 833 
 834     case T_LONG:    // fall through
 835     case T_DOUBLE:
 836 #ifdef _LP64
 837       if (is_literal_address(addr)) {
 838         ShouldNotReachHere();
 839         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 840       } else {
 841         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 842         null_check_here = code_offset();
 843         __ movptr(as_Address_lo(addr), r10);
 844       }
 845 #else
 846       // Always reachable in 32bit so this doesn't produce useless move literal
 847       __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
 848       __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
 849 #endif // _LP64
 850       break;
 851 
 852     case T_BOOLEAN: // fall through
 853     case T_BYTE:
 854       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 855       break;
 856 
 857     case T_CHAR:    // fall through
 858     case T_SHORT:
 859       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 860       break;
 861 
 862     default:
 863       ShouldNotReachHere();
 864   };
 865 
 866   if (info != NULL) {
 867     add_debug_info_for_null_check(null_check_here, info);
 868   }
 869 }
 870 
 871 
 872 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 873   assert(src->is_register(), "should not call otherwise");
 874   assert(dest->is_register(), "should not call otherwise");
 875 
 876   // move between cpu-registers
 877   if (dest->is_single_cpu()) {
 878 #ifdef _LP64
 879     if (src->type() == T_LONG) {
 880       // Can do LONG -> OBJECT
 881       move_regs(src->as_register_lo(), dest->as_register());
 882       return;
 883     }
 884 #endif
 885     assert(src->is_single_cpu(), "must match");
 886     if (src->type() == T_OBJECT || src->type() == T_VALUETYPE) {
 887       __ verify_oop(src->as_register());
 888     }
 889     move_regs(src->as_register(), dest->as_register());
 890 
 891   } else if (dest->is_double_cpu()) {
 892 #ifdef _LP64
 893     if (src->type() == T_OBJECT || src->type() == T_ARRAY || src->type() == T_VALUETYPE) {
 894       // Surprising to me but we can see move of a long to t_object
 895       __ verify_oop(src->as_register());
 896       move_regs(src->as_register(), dest->as_register_lo());
 897       return;
 898     }
 899 #endif
 900     assert(src->is_double_cpu(), "must match");
 901     Register f_lo = src->as_register_lo();
 902     Register f_hi = src->as_register_hi();
 903     Register t_lo = dest->as_register_lo();
 904     Register t_hi = dest->as_register_hi();
 905 #ifdef _LP64
 906     assert(f_hi == f_lo, "must be same");
 907     assert(t_hi == t_lo, "must be same");
 908     move_regs(f_lo, t_lo);
 909 #else
 910     assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
 911 
 912 
 913     if (f_lo == t_hi && f_hi == t_lo) {
 914       swap_reg(f_lo, f_hi);
 915     } else if (f_hi == t_lo) {
 916       assert(f_lo != t_hi, "overwriting register");
 917       move_regs(f_hi, t_hi);
 918       move_regs(f_lo, t_lo);
 919     } else {
 920       assert(f_hi != t_lo, "overwriting register");
 921       move_regs(f_lo, t_lo);
 922       move_regs(f_hi, t_hi);
 923     }
 924 #endif // LP64
 925 
 926     // special moves from fpu-register to xmm-register
 927     // necessary for method results
 928   } else if (src->is_single_xmm() && !dest->is_single_xmm()) {
 929     __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
 930     __ fld_s(Address(rsp, 0));
 931   } else if (src->is_double_xmm() && !dest->is_double_xmm()) {
 932     __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
 933     __ fld_d(Address(rsp, 0));
 934   } else if (dest->is_single_xmm() && !src->is_single_xmm()) {
 935     __ fstp_s(Address(rsp, 0));
 936     __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
 937   } else if (dest->is_double_xmm() && !src->is_double_xmm()) {
 938     __ fstp_d(Address(rsp, 0));
 939     __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
 940 
 941     // move between xmm-registers
 942   } else if (dest->is_single_xmm()) {
 943     assert(src->is_single_xmm(), "must match");
 944     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 945   } else if (dest->is_double_xmm()) {
 946     assert(src->is_double_xmm(), "must match");
 947     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 948 
 949     // move between fpu-registers (no instruction necessary because of fpu-stack)
 950   } else if (dest->is_single_fpu() || dest->is_double_fpu()) {
 951     assert(src->is_single_fpu() || src->is_double_fpu(), "must match");
 952     assert(src->fpu() == dest->fpu(), "currently should be nothing to do");
 953   } else {
 954     ShouldNotReachHere();
 955   }
 956 }
 957 
 958 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 959   assert(src->is_register(), "should not call otherwise");
 960   assert(dest->is_stack(), "should not call otherwise");
 961 
 962   if (src->is_single_cpu()) {
 963     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 964     if (type == T_OBJECT || type == T_ARRAY || type == T_VALUETYPE) {
 965       __ verify_oop(src->as_register());
 966       __ movptr (dst, src->as_register());
 967     } else if (type == T_METADATA) {
 968       __ movptr (dst, src->as_register());
 969     } else {
 970       __ movl (dst, src->as_register());
 971     }
 972 
 973   } else if (src->is_double_cpu()) {
 974     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 975     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 976     __ movptr (dstLO, src->as_register_lo());
 977     NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
 978 
 979   } else if (src->is_single_xmm()) {
 980     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 981     __ movflt(dst_addr, src->as_xmm_float_reg());
 982 
 983   } else if (src->is_double_xmm()) {
 984     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 985     __ movdbl(dst_addr, src->as_xmm_double_reg());
 986 
 987   } else if (src->is_single_fpu()) {
 988     assert(src->fpu_regnr() == 0, "argument must be on TOS");
 989     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 990     if (pop_fpu_stack)     __ fstp_s (dst_addr);
 991     else                   __ fst_s  (dst_addr);
 992 
 993   } else if (src->is_double_fpu()) {
 994     assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
 995     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 996     if (pop_fpu_stack)     __ fstp_d (dst_addr);
 997     else                   __ fst_d  (dst_addr);
 998 
 999   } else {
1000     ShouldNotReachHere();
1001   }
1002 }
1003 
1004 
1005 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
1006   LIR_Address* to_addr = dest->as_address_ptr();
1007   PatchingStub* patch = NULL;
1008   Register compressed_src = rscratch1;
1009 
1010   if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
1011     __ verify_oop(src->as_register());
1012 #ifdef _LP64
1013     if (UseCompressedOops && !wide) {
1014       __ movptr(compressed_src, src->as_register());
1015       __ encode_heap_oop(compressed_src);
1016       if (patch_code != lir_patch_none) {
1017         info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
1018       }
1019     }
1020 #endif
1021   }
1022 
1023   if (patch_code != lir_patch_none) {
1024     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1025     Address toa = as_Address(to_addr);
1026     assert(toa.disp() != 0, "must have");
1027   }
1028 
1029   int null_check_here = code_offset();
1030   switch (type) {
1031     case T_FLOAT: {
1032       if (src->is_single_xmm()) {
1033         __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
1034       } else {
1035         assert(src->is_single_fpu(), "must be");
1036         assert(src->fpu_regnr() == 0, "argument must be on TOS");
1037         if (pop_fpu_stack)      __ fstp_s(as_Address(to_addr));
1038         else                    __ fst_s (as_Address(to_addr));
1039       }
1040       break;
1041     }
1042 
1043     case T_DOUBLE: {
1044       if (src->is_double_xmm()) {
1045         __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
1046       } else {
1047         assert(src->is_double_fpu(), "must be");
1048         assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
1049         if (pop_fpu_stack)      __ fstp_d(as_Address(to_addr));
1050         else                    __ fst_d (as_Address(to_addr));
1051       }
1052       break;
1053     }
1054 
1055     case T_VALUETYPE: // fall through
1056     case T_ARRAY:   // fall through
1057     case T_OBJECT:  // fall through
1058       if (UseCompressedOops && !wide) {
1059         __ movl(as_Address(to_addr), compressed_src);
1060       } else {
1061         __ movptr(as_Address(to_addr), src->as_register());
1062       }
1063       break;
1064     case T_METADATA:
1065       // We get here to store a method pointer to the stack to pass to
1066       // a dtrace runtime call. This can't work on 64 bit with
1067       // compressed klass ptrs: T_METADATA can be a compressed klass
1068       // ptr or a 64 bit method pointer.
1069       LP64_ONLY(ShouldNotReachHere());
1070       __ movptr(as_Address(to_addr), src->as_register());
1071       break;
1072     case T_ADDRESS:
1073       __ movptr(as_Address(to_addr), src->as_register());
1074       break;
1075     case T_INT:
1076       __ movl(as_Address(to_addr), src->as_register());
1077       break;
1078 
1079     case T_LONG: {
1080       Register from_lo = src->as_register_lo();
1081       Register from_hi = src->as_register_hi();
1082 #ifdef _LP64
1083       __ movptr(as_Address_lo(to_addr), from_lo);
1084 #else
1085       Register base = to_addr->base()->as_register();
1086       Register index = noreg;
1087       if (to_addr->index()->is_register()) {
1088         index = to_addr->index()->as_register();
1089       }
1090       if (base == from_lo || index == from_lo) {
1091         assert(base != from_hi, "can't be");
1092         assert(index == noreg || (index != base && index != from_hi), "can't handle this");
1093         __ movl(as_Address_hi(to_addr), from_hi);
1094         if (patch != NULL) {
1095           patching_epilog(patch, lir_patch_high, base, info);
1096           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1097           patch_code = lir_patch_low;
1098         }
1099         __ movl(as_Address_lo(to_addr), from_lo);
1100       } else {
1101         assert(index == noreg || (index != base && index != from_lo), "can't handle this");
1102         __ movl(as_Address_lo(to_addr), from_lo);
1103         if (patch != NULL) {
1104           patching_epilog(patch, lir_patch_low, base, info);
1105           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1106           patch_code = lir_patch_high;
1107         }
1108         __ movl(as_Address_hi(to_addr), from_hi);
1109       }
1110 #endif // _LP64
1111       break;
1112     }
1113 
1114     case T_BYTE:    // fall through
1115     case T_BOOLEAN: {
1116       Register src_reg = src->as_register();
1117       Address dst_addr = as_Address(to_addr);
1118       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
1119       __ movb(dst_addr, src_reg);
1120       break;
1121     }
1122 
1123     case T_CHAR:    // fall through
1124     case T_SHORT:
1125       __ movw(as_Address(to_addr), src->as_register());
1126       break;
1127 
1128     default:
1129       ShouldNotReachHere();
1130   }
1131   if (info != NULL) {
1132     add_debug_info_for_null_check(null_check_here, info);
1133   }
1134 
1135   if (patch_code != lir_patch_none) {
1136     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1137   }
1138 }
1139 
1140 
1141 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1142   assert(src->is_stack(), "should not call otherwise");
1143   assert(dest->is_register(), "should not call otherwise");
1144 
1145   if (dest->is_single_cpu()) {
1146     if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
1147       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1148       __ verify_oop(dest->as_register());
1149     } else if (type == T_METADATA) {
1150       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1151     } else {
1152       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1153     }
1154 
1155   } else if (dest->is_double_cpu()) {
1156     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1157     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1158     __ movptr(dest->as_register_lo(), src_addr_LO);
1159     NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1160 
1161   } else if (dest->is_single_xmm()) {
1162     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1163     __ movflt(dest->as_xmm_float_reg(), src_addr);
1164 
1165   } else if (dest->is_double_xmm()) {
1166     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1167     __ movdbl(dest->as_xmm_double_reg(), src_addr);
1168 
1169   } else if (dest->is_single_fpu()) {
1170     assert(dest->fpu_regnr() == 0, "dest must be TOS");
1171     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1172     __ fld_s(src_addr);
1173 
1174   } else if (dest->is_double_fpu()) {
1175     assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1176     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1177     __ fld_d(src_addr);
1178 
1179   } else {
1180     ShouldNotReachHere();
1181   }
1182 }
1183 
1184 
1185 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1186   if (src->is_single_stack()) {
1187     if (type == T_OBJECT || type == T_ARRAY || type == T_VALUETYPE) {
1188       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1189       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1190     } else {
1191 #ifndef _LP64
1192       __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1193       __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1194 #else
1195       //no pushl on 64bits
1196       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1197       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1198 #endif
1199     }
1200 
1201   } else if (src->is_double_stack()) {
1202 #ifdef _LP64
1203     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1204     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1205 #else
1206     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1207     // push and pop the part at src + wordSize, adding wordSize for the previous push
1208     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1209     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1210     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1211 #endif // _LP64
1212 
1213   } else {
1214     ShouldNotReachHere();
1215   }
1216 }
1217 
1218 
1219 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
1220   assert(src->is_address(), "should not call otherwise");
1221   assert(dest->is_register(), "should not call otherwise");
1222 
1223   LIR_Address* addr = src->as_address_ptr();
1224   Address from_addr = as_Address(addr);
1225 
1226   if (addr->base()->type() == T_OBJECT || addr->base()->type() == T_VALUETYPE) {
1227     __ verify_oop(addr->base()->as_pointer_register());
1228   }
1229 
1230   switch (type) {
1231     case T_BOOLEAN: // fall through
1232     case T_BYTE:    // fall through
1233     case T_CHAR:    // fall through
1234     case T_SHORT:
1235       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1236         // on pre P6 processors we may get partial register stalls
1237         // so blow away the value of to_rinfo before loading a
1238         // partial word into it.  Do it here so that it precedes
1239         // the potential patch point below.
1240         __ xorptr(dest->as_register(), dest->as_register());
1241       }
1242       break;
1243    default:
1244      break;
1245   }
1246 
1247   PatchingStub* patch = NULL;
1248   if (patch_code != lir_patch_none) {
1249     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1250     assert(from_addr.disp() != 0, "must have");
1251   }
1252   if (info != NULL) {
1253     add_debug_info_for_null_check_here(info);
1254   }
1255 
1256   switch (type) {
1257     case T_FLOAT: {
1258       if (dest->is_single_xmm()) {
1259         __ movflt(dest->as_xmm_float_reg(), from_addr);
1260       } else {
1261         assert(dest->is_single_fpu(), "must be");
1262         assert(dest->fpu_regnr() == 0, "dest must be TOS");
1263         __ fld_s(from_addr);
1264       }
1265       break;
1266     }
1267 
1268     case T_DOUBLE: {
1269       if (dest->is_double_xmm()) {
1270         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1271       } else {
1272         assert(dest->is_double_fpu(), "must be");
1273         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1274         __ fld_d(from_addr);
1275       }
1276       break;
1277     }
1278 
1279     case T_VALUETYPE: // fall through
1280     case T_OBJECT:  // fall through
1281     case T_ARRAY:   // fall through
1282       if (UseCompressedOops && !wide) {
1283         __ movl(dest->as_register(), from_addr);
1284       } else {
1285         __ movptr(dest->as_register(), from_addr);
1286       }
1287       break;
1288 
1289     case T_ADDRESS:
1290       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1291         __ movl(dest->as_register(), from_addr);
1292       } else {
1293         __ movptr(dest->as_register(), from_addr);
1294       }
1295       break;
1296     case T_INT:
1297       __ movl(dest->as_register(), from_addr);
1298       break;
1299 
1300     case T_LONG: {
1301       Register to_lo = dest->as_register_lo();
1302       Register to_hi = dest->as_register_hi();
1303 #ifdef _LP64
1304       __ movptr(to_lo, as_Address_lo(addr));
1305 #else
1306       Register base = addr->base()->as_register();
1307       Register index = noreg;
1308       if (addr->index()->is_register()) {
1309         index = addr->index()->as_register();
1310       }
1311       if ((base == to_lo && index == to_hi) ||
1312           (base == to_hi && index == to_lo)) {
1313         // addresses with 2 registers are only formed as a result of
1314         // array access so this code will never have to deal with
1315         // patches or null checks.
1316         assert(info == NULL && patch == NULL, "must be");
1317         __ lea(to_hi, as_Address(addr));
1318         __ movl(to_lo, Address(to_hi, 0));
1319         __ movl(to_hi, Address(to_hi, BytesPerWord));
1320       } else if (base == to_lo || index == to_lo) {
1321         assert(base != to_hi, "can't be");
1322         assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1323         __ movl(to_hi, as_Address_hi(addr));
1324         if (patch != NULL) {
1325           patching_epilog(patch, lir_patch_high, base, info);
1326           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1327           patch_code = lir_patch_low;
1328         }
1329         __ movl(to_lo, as_Address_lo(addr));
1330       } else {
1331         assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1332         __ movl(to_lo, as_Address_lo(addr));
1333         if (patch != NULL) {
1334           patching_epilog(patch, lir_patch_low, base, info);
1335           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1336           patch_code = lir_patch_high;
1337         }
1338         __ movl(to_hi, as_Address_hi(addr));
1339       }
1340 #endif // _LP64
1341       break;
1342     }
1343 
1344     case T_BOOLEAN: // fall through
1345     case T_BYTE: {
1346       Register dest_reg = dest->as_register();
1347       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1348       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1349         __ movsbl(dest_reg, from_addr);
1350       } else {
1351         __ movb(dest_reg, from_addr);
1352         __ shll(dest_reg, 24);
1353         __ sarl(dest_reg, 24);
1354       }
1355       break;
1356     }
1357 
1358     case T_CHAR: {
1359       Register dest_reg = dest->as_register();
1360       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1361       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1362         __ movzwl(dest_reg, from_addr);
1363       } else {
1364         __ movw(dest_reg, from_addr);
1365       }
1366       break;
1367     }
1368 
1369     case T_SHORT: {
1370       Register dest_reg = dest->as_register();
1371       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1372         __ movswl(dest_reg, from_addr);
1373       } else {
1374         __ movw(dest_reg, from_addr);
1375         __ shll(dest_reg, 16);
1376         __ sarl(dest_reg, 16);
1377       }
1378       break;
1379     }
1380 
1381     default:
1382       ShouldNotReachHere();
1383   }
1384 
1385   if (patch != NULL) {
1386     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1387   }
1388 
1389   if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
1390 #ifdef _LP64
1391     if (UseCompressedOops && !wide) {
1392       __ decode_heap_oop(dest->as_register());
1393     }
1394 #endif
1395 
1396     // Load barrier has not yet been applied, so ZGC can't verify the oop here
1397     if (!UseZGC) {
1398       __ verify_oop(dest->as_register());
1399     }
1400   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1401 #ifdef _LP64
1402     if (UseCompressedClassPointers) {
1403       __ andl(dest->as_register(), oopDesc::compressed_klass_mask());
1404       __ decode_klass_not_null(dest->as_register());
1405     } else {
1406       __ shlq(dest->as_register(), oopDesc::storage_props_nof_bits);
1407       __ shrq(dest->as_register(), oopDesc::storage_props_nof_bits);
1408     }
1409 #else
1410     __ andl(dest->as_register(), oopDesc::wide_klass_mask());
1411 #endif
1412   }
1413 }
1414 
1415 
1416 NEEDS_CLEANUP; // This could be static?
1417 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1418   int elem_size = type2aelembytes(type);
1419   switch (elem_size) {
1420     case 1: return Address::times_1;
1421     case 2: return Address::times_2;
1422     case 4: return Address::times_4;
1423     case 8: return Address::times_8;
1424   }
1425   ShouldNotReachHere();
1426   return Address::no_scale;
1427 }
1428 
1429 
1430 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1431   switch (op->code()) {
1432     case lir_idiv:
1433     case lir_irem:
1434       arithmetic_idiv(op->code(),
1435                       op->in_opr1(),
1436                       op->in_opr2(),
1437                       op->in_opr3(),
1438                       op->result_opr(),
1439                       op->info());
1440       break;
1441     case lir_fmad:
1442       __ fmad(op->result_opr()->as_xmm_double_reg(),
1443               op->in_opr1()->as_xmm_double_reg(),
1444               op->in_opr2()->as_xmm_double_reg(),
1445               op->in_opr3()->as_xmm_double_reg());
1446       break;
1447     case lir_fmaf:
1448       __ fmaf(op->result_opr()->as_xmm_float_reg(),
1449               op->in_opr1()->as_xmm_float_reg(),
1450               op->in_opr2()->as_xmm_float_reg(),
1451               op->in_opr3()->as_xmm_float_reg());
1452       break;
1453     default:      ShouldNotReachHere(); break;
1454   }
1455 }
1456 
1457 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1458 #ifdef ASSERT
1459   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1460   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1461   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1462 #endif
1463 
1464   if (op->cond() == lir_cond_always) {
1465     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1466     __ jmp (*(op->label()));
1467   } else {
1468     Assembler::Condition acond = Assembler::zero;
1469     if (op->code() == lir_cond_float_branch) {
1470       assert(op->ublock() != NULL, "must have unordered successor");
1471       __ jcc(Assembler::parity, *(op->ublock()->label()));
1472       switch(op->cond()) {
1473         case lir_cond_equal:        acond = Assembler::equal;      break;
1474         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1475         case lir_cond_less:         acond = Assembler::below;      break;
1476         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1477         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1478         case lir_cond_greater:      acond = Assembler::above;      break;
1479         default:                         ShouldNotReachHere();
1480       }
1481     } else {
1482       switch (op->cond()) {
1483         case lir_cond_equal:        acond = Assembler::equal;       break;
1484         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1485         case lir_cond_less:         acond = Assembler::less;        break;
1486         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1487         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1488         case lir_cond_greater:      acond = Assembler::greater;     break;
1489         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1490         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1491         default:                         ShouldNotReachHere();
1492       }
1493     }
1494     __ jcc(acond,*(op->label()));
1495   }
1496 }
1497 
1498 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1499   LIR_Opr src  = op->in_opr();
1500   LIR_Opr dest = op->result_opr();
1501 
1502   switch (op->bytecode()) {
1503     case Bytecodes::_i2l:
1504 #ifdef _LP64
1505       __ movl2ptr(dest->as_register_lo(), src->as_register());
1506 #else
1507       move_regs(src->as_register(), dest->as_register_lo());
1508       move_regs(src->as_register(), dest->as_register_hi());
1509       __ sarl(dest->as_register_hi(), 31);
1510 #endif // LP64
1511       break;
1512 
1513     case Bytecodes::_l2i:
1514 #ifdef _LP64
1515       __ movl(dest->as_register(), src->as_register_lo());
1516 #else
1517       move_regs(src->as_register_lo(), dest->as_register());
1518 #endif
1519       break;
1520 
1521     case Bytecodes::_i2b:
1522       move_regs(src->as_register(), dest->as_register());
1523       __ sign_extend_byte(dest->as_register());
1524       break;
1525 
1526     case Bytecodes::_i2c:
1527       move_regs(src->as_register(), dest->as_register());
1528       __ andl(dest->as_register(), 0xFFFF);
1529       break;
1530 
1531     case Bytecodes::_i2s:
1532       move_regs(src->as_register(), dest->as_register());
1533       __ sign_extend_short(dest->as_register());
1534       break;
1535 
1536 
1537     case Bytecodes::_f2d:
1538     case Bytecodes::_d2f:
1539       if (dest->is_single_xmm()) {
1540         __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1541       } else if (dest->is_double_xmm()) {
1542         __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1543       } else {
1544         assert(src->fpu() == dest->fpu(), "register must be equal");
1545         // do nothing (float result is rounded later through spilling)
1546       }
1547       break;
1548 
1549     case Bytecodes::_i2f:
1550     case Bytecodes::_i2d:
1551       if (dest->is_single_xmm()) {
1552         __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1553       } else if (dest->is_double_xmm()) {
1554         __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1555       } else {
1556         assert(dest->fpu() == 0, "result must be on TOS");
1557         __ movl(Address(rsp, 0), src->as_register());
1558         __ fild_s(Address(rsp, 0));
1559       }
1560       break;
1561 
1562     case Bytecodes::_f2i:
1563     case Bytecodes::_d2i:
1564       if (src->is_single_xmm()) {
1565         __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1566       } else if (src->is_double_xmm()) {
1567         __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1568       } else {
1569         assert(src->fpu() == 0, "input must be on TOS");
1570         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
1571         __ fist_s(Address(rsp, 0));
1572         __ movl(dest->as_register(), Address(rsp, 0));
1573         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1574       }
1575 
1576       // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
1577       assert(op->stub() != NULL, "stub required");
1578       __ cmpl(dest->as_register(), 0x80000000);
1579       __ jcc(Assembler::equal, *op->stub()->entry());
1580       __ bind(*op->stub()->continuation());
1581       break;
1582 
1583     case Bytecodes::_l2f:
1584     case Bytecodes::_l2d:
1585       assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)");
1586       assert(dest->fpu() == 0, "result must be on TOS");
1587 
1588       __ movptr(Address(rsp, 0),            src->as_register_lo());
1589       NOT_LP64(__ movl(Address(rsp, BytesPerWord), src->as_register_hi()));
1590       __ fild_d(Address(rsp, 0));
1591       // float result is rounded later through spilling
1592       break;
1593 
1594     case Bytecodes::_f2l:
1595     case Bytecodes::_d2l:
1596       assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)");
1597       assert(src->fpu() == 0, "input must be on TOS");
1598       assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers");
1599 
1600       // instruction sequence too long to inline it here
1601       {
1602         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1603       }
1604       break;
1605 
1606     default: ShouldNotReachHere();
1607   }
1608 }
1609 
1610 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1611   if (op->init_check()) {
1612     add_debug_info_for_null_check_here(op->stub()->info());
1613     __ cmpb(Address(op->klass()->as_register(),
1614                     InstanceKlass::init_state_offset()),
1615                     InstanceKlass::fully_initialized);
1616     __ jcc(Assembler::notEqual, *op->stub()->entry());
1617   }
1618   __ allocate_object(op->obj()->as_register(),
1619                      op->tmp1()->as_register(),
1620                      op->tmp2()->as_register(),
1621                      op->header_size(),
1622                      op->object_size(),
1623                      op->klass()->as_register(),
1624                      *op->stub()->entry());
1625   __ bind(*op->stub()->continuation());
1626 }
1627 
1628 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1629   Register len =  op->len()->as_register();
1630   LP64_ONLY( __ movslq(len, len); )
1631 
1632   if (UseSlowPath || op->type() == T_VALUETYPE ||
1633       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1634       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1635     __ jmp(*op->stub()->entry());
1636   } else {
1637     Register tmp1 = op->tmp1()->as_register();
1638     Register tmp2 = op->tmp2()->as_register();
1639     Register tmp3 = op->tmp3()->as_register();
1640     if (len == tmp1) {
1641       tmp1 = tmp3;
1642     } else if (len == tmp2) {
1643       tmp2 = tmp3;
1644     } else if (len == tmp3) {
1645       // everything is ok
1646     } else {
1647       __ mov(tmp3, len);
1648     }
1649     __ allocate_array(op->obj()->as_register(),
1650                       len,
1651                       tmp1,
1652                       tmp2,
1653                       arrayOopDesc::header_size(op->type()),
1654                       array_element_size(op->type()),
1655                       op->klass()->as_register(),
1656                       *op->stub()->entry());
1657   }
1658   __ bind(*op->stub()->continuation());
1659 }
1660 
1661 void LIR_Assembler::type_profile_helper(Register mdo,
1662                                         ciMethodData *md, ciProfileData *data,
1663                                         Register recv, Label* update_done) {
1664   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1665     Label next_test;
1666     // See if the receiver is receiver[n].
1667     __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1668     __ jccb(Assembler::notEqual, next_test);
1669     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1670     __ addptr(data_addr, DataLayout::counter_increment);
1671     __ jmp(*update_done);
1672     __ bind(next_test);
1673   }
1674 
1675   // Didn't find receiver; find next empty slot and fill it in
1676   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1677     Label next_test;
1678     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1679     __ cmpptr(recv_addr, (intptr_t)NULL_WORD);
1680     __ jccb(Assembler::notEqual, next_test);
1681     __ movptr(recv_addr, recv);
1682     __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1683     __ jmp(*update_done);
1684     __ bind(next_test);
1685   }
1686 }
1687 
1688 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1689   // we always need a stub for the failure case.
1690   CodeStub* stub = op->stub();
1691   Register obj = op->object()->as_register();
1692   Register k_RInfo = op->tmp1()->as_register();
1693   Register klass_RInfo = op->tmp2()->as_register();
1694   Register dst = op->result_opr()->as_register();
1695   ciKlass* k = op->klass();
1696   Register Rtmp1 = noreg;
1697 
1698   // check if it needs to be profiled
1699   ciMethodData* md = NULL;
1700   ciProfileData* data = NULL;
1701 
1702   if (op->should_profile()) {
1703     ciMethod* method = op->profiled_method();
1704     assert(method != NULL, "Should have method");
1705     int bci = op->profiled_bci();
1706     md = method->method_data_or_null();
1707     assert(md != NULL, "Sanity");
1708     data = md->bci_to_data(bci);
1709     assert(data != NULL,                "need data for type check");
1710     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1711   }
1712   Label profile_cast_success, profile_cast_failure;
1713   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1714   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1715 
1716   if (obj == k_RInfo) {
1717     k_RInfo = dst;
1718   } else if (obj == klass_RInfo) {
1719     klass_RInfo = dst;
1720   }
1721   if (k->is_loaded() && !UseCompressedClassPointers) {
1722     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1723   } else {
1724     Rtmp1 = op->tmp3()->as_register();
1725     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1726   }
1727 
1728   assert_different_registers(obj, k_RInfo, klass_RInfo);
1729 
1730   if (op->need_null_check()) {
1731     __ cmpptr(obj, (int32_t)NULL_WORD);
1732     if (op->should_profile()) {
1733       Label not_null;
1734       __ jccb(Assembler::notEqual, not_null);
1735       // Object is null; update MDO and exit
1736       Register mdo  = klass_RInfo;
1737       __ mov_metadata(mdo, md->constant_encoding());
1738       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1739       int header_bits = BitData::null_seen_byte_constant();
1740       __ orb(data_addr, header_bits);
1741       __ jmp(*obj_is_null);
1742       __ bind(not_null);
1743     } else {
1744       __ jcc(Assembler::equal, *obj_is_null);
1745     }
1746   }
1747 
1748   if (!k->is_loaded()) {
1749     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1750   } else {
1751 #ifdef _LP64
1752     __ mov_metadata(k_RInfo, k->constant_encoding());
1753 #endif // _LP64
1754   }
1755   __ verify_oop(obj);
1756 
1757   if (op->fast_check()) {
1758     // get object class
1759     // not a safepoint as obj null check happens earlier
1760 #ifdef _LP64
1761     if (UseCompressedClassPointers) {
1762       __ load_klass(Rtmp1, obj);
1763       __ cmpptr(k_RInfo, Rtmp1);
1764     } else {
1765       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1766     }
1767 #else
1768     if (k->is_loaded()) {
1769       __ cmpklass(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1770     } else {
1771       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1772     }
1773 #endif
1774     __ jcc(Assembler::notEqual, *failure_target);
1775     // successful cast, fall through to profile or jump
1776   } else {
1777     // get object class
1778     // not a safepoint as obj null check happens earlier
1779     __ load_klass(klass_RInfo, obj);
1780     if (k->is_loaded()) {
1781       // See if we get an immediate positive hit
1782 #ifdef _LP64
1783       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1784 #else
1785       __ cmpklass(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1786 #endif // _LP64
1787       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1788         __ jcc(Assembler::notEqual, *failure_target);
1789         // successful cast, fall through to profile or jump
1790       } else {
1791         // See if we get an immediate positive hit
1792         __ jcc(Assembler::equal, *success_target);
1793         // check for self
1794 #ifdef _LP64
1795         __ cmpptr(klass_RInfo, k_RInfo);
1796 #else
1797         __ cmpklass(klass_RInfo, k->constant_encoding());
1798 #endif // _LP64
1799         __ jcc(Assembler::equal, *success_target);
1800 
1801         __ push(klass_RInfo);
1802 #ifdef _LP64
1803         __ push(k_RInfo);
1804 #else
1805         __ pushklass(k->constant_encoding());
1806 #endif // _LP64
1807         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1808         __ pop(klass_RInfo);
1809         __ pop(klass_RInfo);
1810         // result is a boolean
1811         __ cmpl(klass_RInfo, 0);
1812         __ jcc(Assembler::equal, *failure_target);
1813         // successful cast, fall through to profile or jump
1814       }
1815     } else {
1816       // perform the fast part of the checking logic
1817       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1818       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1819       __ push(klass_RInfo);
1820       __ push(k_RInfo);
1821       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1822       __ pop(klass_RInfo);
1823       __ pop(k_RInfo);
1824       // result is a boolean
1825       __ cmpl(k_RInfo, 0);
1826       __ jcc(Assembler::equal, *failure_target);
1827       // successful cast, fall through to profile or jump
1828     }
1829   }
1830   if (op->should_profile()) {
1831     Register mdo  = klass_RInfo, recv = k_RInfo;
1832     __ bind(profile_cast_success);
1833     __ mov_metadata(mdo, md->constant_encoding());
1834     __ load_klass(recv, obj);
1835     type_profile_helper(mdo, md, data, recv, success);
1836     __ jmp(*success);
1837 
1838     __ bind(profile_cast_failure);
1839     __ mov_metadata(mdo, md->constant_encoding());
1840     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1841     __ subptr(counter_addr, DataLayout::counter_increment);
1842     __ jmp(*failure);
1843   }
1844   __ jmp(*success);
1845 }
1846 
1847 
1848 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1849   LIR_Code code = op->code();
1850   if (code == lir_store_check) {
1851     Register value = op->object()->as_register();
1852     Register array = op->array()->as_register();
1853     Register k_RInfo = op->tmp1()->as_register();
1854     Register klass_RInfo = op->tmp2()->as_register();
1855     Register Rtmp1 = op->tmp3()->as_register();
1856 
1857     CodeStub* stub = op->stub();
1858 
1859     // check if it needs to be profiled
1860     ciMethodData* md = NULL;
1861     ciProfileData* data = NULL;
1862 
1863     if (op->should_profile()) {
1864       ciMethod* method = op->profiled_method();
1865       assert(method != NULL, "Should have method");
1866       int bci = op->profiled_bci();
1867       md = method->method_data_or_null();
1868       assert(md != NULL, "Sanity");
1869       data = md->bci_to_data(bci);
1870       assert(data != NULL,                "need data for type check");
1871       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1872     }
1873     Label profile_cast_success, profile_cast_failure, done;
1874     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1875     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1876 
1877     __ cmpptr(value, (int32_t)NULL_WORD);
1878     if (op->should_profile()) {
1879       Label not_null;
1880       __ jccb(Assembler::notEqual, not_null);
1881       // Object is null; update MDO and exit
1882       Register mdo  = klass_RInfo;
1883       __ mov_metadata(mdo, md->constant_encoding());
1884       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1885       int header_bits = BitData::null_seen_byte_constant();
1886       __ orb(data_addr, header_bits);
1887       __ jmp(done);
1888       __ bind(not_null);
1889     } else {
1890       __ jcc(Assembler::equal, done);
1891     }
1892 
1893     add_debug_info_for_null_check_here(op->info_for_exception());
1894     __ load_klass(k_RInfo, array);
1895     __ load_klass(klass_RInfo, value);
1896 
1897     // get instance klass (it's already uncompressed)
1898     __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1899     // perform the fast part of the checking logic
1900     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1901     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1902     __ push(klass_RInfo);
1903     __ push(k_RInfo);
1904     __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1905     __ pop(klass_RInfo);
1906     __ pop(k_RInfo);
1907     // result is a boolean
1908     __ cmpl(k_RInfo, 0);
1909     __ jcc(Assembler::equal, *failure_target);
1910     // fall through to the success case
1911 
1912     if (op->should_profile()) {
1913       Register mdo  = klass_RInfo, recv = k_RInfo;
1914       __ bind(profile_cast_success);
1915       __ mov_metadata(mdo, md->constant_encoding());
1916       __ load_klass(recv, value);
1917       type_profile_helper(mdo, md, data, recv, &done);
1918       __ jmpb(done);
1919 
1920       __ bind(profile_cast_failure);
1921       __ mov_metadata(mdo, md->constant_encoding());
1922       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1923       __ subptr(counter_addr, DataLayout::counter_increment);
1924       __ jmp(*stub->entry());
1925     }
1926 
1927     __ bind(done);
1928   } else
1929     if (code == lir_checkcast) {
1930       Register obj = op->object()->as_register();
1931       Register dst = op->result_opr()->as_register();
1932       Label success;
1933       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1934       __ bind(success);
1935       if (dst != obj) {
1936         __ mov(dst, obj);
1937       }
1938     } else
1939       if (code == lir_instanceof) {
1940         Register obj = op->object()->as_register();
1941         Register dst = op->result_opr()->as_register();
1942         Label success, failure, done;
1943         emit_typecheck_helper(op, &success, &failure, &failure);
1944         __ bind(failure);
1945         __ xorptr(dst, dst);
1946         __ jmpb(done);
1947         __ bind(success);
1948         __ movptr(dst, 1);
1949         __ bind(done);
1950       } else {
1951         ShouldNotReachHere();
1952       }
1953 
1954 }
1955 
1956 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1957   // We are loading/storing an array that *may* be a flattened array (the declared type
1958   // Object[], interface[], or VT?[]). If this array is flattened, take slow path.
1959 
1960   __ load_storage_props(op->tmp()->as_register(), op->array()->as_register());
1961   __ testb(op->tmp()->as_register(), ArrayStorageProperties::flattened_value);
1962   __ jcc(Assembler::notZero, *op->stub()->entry());
1963   if (!op->value()->is_illegal()) {
1964     // We are storing into the array.
1965     Label skip;
1966     __ testb(op->tmp()->as_register(), ArrayStorageProperties::null_free_value);
1967     __ jcc(Assembler::zero, skip);
1968     // The array is not flattened, but it is null_free. If we are storing
1969     // a null, take the slow path (which will throw NPE).
1970     __ cmpptr(op->value()->as_register(), (int32_t)NULL_WORD);
1971     __ jcc(Assembler::zero, *op->stub()->entry());
1972     __ bind(skip);
1973   }
1974 }
1975 
1976 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1977   // This is called when we use aastore into a an array declared as "[LVT;",
1978   // where we know VT is not flattenable (due to ValueArrayElemMaxFlatOops, etc).
1979   // However, we need to do a NULL check if the actual array is a "[QVT;".
1980 
1981   __ load_storage_props(op->tmp()->as_register(), op->array()->as_register());
1982   __ testb(op->tmp()->as_register(), ArrayStorageProperties::null_free_value);
1983 }
1984 
1985 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1986   Label L_oops_equal;
1987   Label L_oops_not_equal;
1988   Label L_end;
1989 
1990   Register left  = op->left()->as_register();
1991   Register right = op->right()->as_register();
1992 
1993   __ cmpptr(left, right);
1994   __ jcc(Assembler::equal, L_oops_equal);
1995 
1996   // (1) Null check -- if one of the operands is null, the other must not be null (because
1997   //     the two references are not equal), so they are not substitutable,
1998   //     FIXME: do null check only if the operand is nullable
1999   {
2000     __ cmpptr(left, (int32_t)NULL_WORD);
2001     __ jcc(Assembler::equal, L_oops_not_equal);
2002 
2003     __ cmpptr(right, (int32_t)NULL_WORD);
2004     __ jcc(Assembler::equal, L_oops_not_equal);
2005   }
2006 
2007   ciKlass* left_klass = op->left_klass();
2008   ciKlass* right_klass = op->right_klass();
2009 
2010   // (2) Value object check -- if either of the operands is not a value object,
2011   //     they are not substitutable. We do this only if we are not sure that the
2012   //     operands are value objects
2013   if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
2014       !left_klass->is_valuetype() || !right_klass->is_valuetype()) {
2015     Register tmp1  = op->tmp1()->as_register();
2016     __ movptr(tmp1, (intptr_t)markWord::always_locked_pattern);
2017     __ andl(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
2018     __ andl(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
2019     __ cmpptr(tmp1, (intptr_t)markWord::always_locked_pattern);
2020     __ jcc(Assembler::notEqual, L_oops_not_equal);
2021   }
2022 
2023   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
2024   if (left_klass != NULL && left_klass->is_valuetype() && left_klass == right_klass) {
2025     // No need to load klass -- the operands are statically known to be the same value klass.
2026     __ jmp(*op->stub()->entry());
2027   } else {
2028     Register left_klass_op = op->left_klass_op()->as_register();
2029     Register right_klass_op = op->right_klass_op()->as_register();
2030 
2031     if (UseCompressedOops) {
2032       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
2033       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2034       __ cmpl(left_klass_op, right_klass_op);
2035     } else {
2036       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
2037       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2038       __ cmpptr(left_klass_op, right_klass_op);
2039     }
2040 
2041     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
2042     // fall through to L_oops_not_equal
2043   }
2044 
2045   __ bind(L_oops_not_equal);
2046   move(op->not_equal_result(), op->result_opr());
2047   __ jmp(L_end);
2048 
2049   __ bind(L_oops_equal);
2050   move(op->equal_result(), op->result_opr());
2051   __ jmp(L_end);
2052 
2053   // We've returned from the stub. RAX contains 0x0 IFF the two
2054   // operands are not substitutable. (Don't compare against 0x1 in case the
2055   // C compiler is naughty)
2056   __ bind(*op->stub()->continuation());
2057   __ cmpl(rax, 0);
2058   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
2059   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
2060   // fall-through
2061   __ bind(L_end);
2062 }
2063 
2064 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2065   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
2066     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
2067     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
2068     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
2069     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
2070     Register addr = op->addr()->as_register();
2071     __ lock();
2072     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
2073 
2074   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
2075     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
2076     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2077     Register newval = op->new_value()->as_register();
2078     Register cmpval = op->cmp_value()->as_register();
2079     assert(cmpval == rax, "wrong register");
2080     assert(newval != NULL, "new val must be register");
2081     assert(cmpval != newval, "cmp and new values must be in different registers");
2082     assert(cmpval != addr, "cmp and addr must be in different registers");
2083     assert(newval != addr, "new value and addr must be in different registers");
2084 
2085     if ( op->code() == lir_cas_obj) {
2086 #ifdef _LP64
2087       if (UseCompressedOops) {
2088         __ encode_heap_oop(cmpval);
2089         __ mov(rscratch1, newval);
2090         __ encode_heap_oop(rscratch1);
2091         __ lock();
2092         // cmpval (rax) is implicitly used by this instruction
2093         __ cmpxchgl(rscratch1, Address(addr, 0));
2094       } else
2095 #endif
2096       {
2097         __ lock();
2098         __ cmpxchgptr(newval, Address(addr, 0));
2099       }
2100     } else {
2101       assert(op->code() == lir_cas_int, "lir_cas_int expected");
2102       __ lock();
2103       __ cmpxchgl(newval, Address(addr, 0));
2104     }
2105 #ifdef _LP64
2106   } else if (op->code() == lir_cas_long) {
2107     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2108     Register newval = op->new_value()->as_register_lo();
2109     Register cmpval = op->cmp_value()->as_register_lo();
2110     assert(cmpval == rax, "wrong register");
2111     assert(newval != NULL, "new val must be register");
2112     assert(cmpval != newval, "cmp and new values must be in different registers");
2113     assert(cmpval != addr, "cmp and addr must be in different registers");
2114     assert(newval != addr, "new value and addr must be in different registers");
2115     __ lock();
2116     __ cmpxchgq(newval, Address(addr, 0));
2117 #endif // _LP64
2118   } else {
2119     Unimplemented();
2120   }
2121 }
2122 
2123 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
2124   assert(dst->is_cpu_register(), "must be");
2125   assert(dst->type() == src->type(), "must be");
2126 
2127   if (src->is_cpu_register()) {
2128     reg2reg(src, dst);
2129   } else if (src->is_stack()) {
2130     stack2reg(src, dst, dst->type());
2131   } else if (src->is_constant()) {
2132     const2reg(src, dst, lir_patch_none, NULL);
2133   } else {
2134     ShouldNotReachHere();
2135   }
2136 }
2137 
2138 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
2139   Assembler::Condition acond, ncond;
2140   switch (condition) {
2141     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
2142     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
2143     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
2144     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
2145     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
2146     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
2147     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
2148     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
2149     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
2150                                 ShouldNotReachHere();
2151   }
2152 
2153   if (opr1->is_cpu_register()) {
2154     reg2reg(opr1, result);
2155   } else if (opr1->is_stack()) {
2156     stack2reg(opr1, result, result->type());
2157   } else if (opr1->is_constant()) {
2158     const2reg(opr1, result, lir_patch_none, NULL);
2159   } else {
2160     ShouldNotReachHere();
2161   }
2162 
2163   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
2164     // optimized version that does not require a branch
2165     if (opr2->is_single_cpu()) {
2166       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
2167       __ cmov(ncond, result->as_register(), opr2->as_register());
2168     } else if (opr2->is_double_cpu()) {
2169       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2170       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2171       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
2172       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
2173     } else if (opr2->is_single_stack()) {
2174       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
2175     } else if (opr2->is_double_stack()) {
2176       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
2177       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
2178     } else {
2179       ShouldNotReachHere();
2180     }
2181 
2182   } else {
2183     Label skip;
2184     __ jcc (acond, skip);
2185     if (opr2->is_cpu_register()) {
2186       reg2reg(opr2, result);
2187     } else if (opr2->is_stack()) {
2188       stack2reg(opr2, result, result->type());
2189     } else if (opr2->is_constant()) {
2190       const2reg(opr2, result, lir_patch_none, NULL);
2191     } else {
2192       ShouldNotReachHere();
2193     }
2194     __ bind(skip);
2195   }
2196 }
2197 
2198 
2199 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
2200   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
2201 
2202   if (left->is_single_cpu()) {
2203     assert(left == dest, "left and dest must be equal");
2204     Register lreg = left->as_register();
2205 
2206     if (right->is_single_cpu()) {
2207       // cpu register - cpu register
2208       Register rreg = right->as_register();
2209       switch (code) {
2210         case lir_add: __ addl (lreg, rreg); break;
2211         case lir_sub: __ subl (lreg, rreg); break;
2212         case lir_mul: __ imull(lreg, rreg); break;
2213         default:      ShouldNotReachHere();
2214       }
2215 
2216     } else if (right->is_stack()) {
2217       // cpu register - stack
2218       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2219       switch (code) {
2220         case lir_add: __ addl(lreg, raddr); break;
2221         case lir_sub: __ subl(lreg, raddr); break;
2222         default:      ShouldNotReachHere();
2223       }
2224 
2225     } else if (right->is_constant()) {
2226       // cpu register - constant
2227       jint c = right->as_constant_ptr()->as_jint();
2228       switch (code) {
2229         case lir_add: {
2230           __ incrementl(lreg, c);
2231           break;
2232         }
2233         case lir_sub: {
2234           __ decrementl(lreg, c);
2235           break;
2236         }
2237         default: ShouldNotReachHere();
2238       }
2239 
2240     } else {
2241       ShouldNotReachHere();
2242     }
2243 
2244   } else if (left->is_double_cpu()) {
2245     assert(left == dest, "left and dest must be equal");
2246     Register lreg_lo = left->as_register_lo();
2247     Register lreg_hi = left->as_register_hi();
2248 
2249     if (right->is_double_cpu()) {
2250       // cpu register - cpu register
2251       Register rreg_lo = right->as_register_lo();
2252       Register rreg_hi = right->as_register_hi();
2253       NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
2254       LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
2255       switch (code) {
2256         case lir_add:
2257           __ addptr(lreg_lo, rreg_lo);
2258           NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2259           break;
2260         case lir_sub:
2261           __ subptr(lreg_lo, rreg_lo);
2262           NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2263           break;
2264         case lir_mul:
2265 #ifdef _LP64
2266           __ imulq(lreg_lo, rreg_lo);
2267 #else
2268           assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2269           __ imull(lreg_hi, rreg_lo);
2270           __ imull(rreg_hi, lreg_lo);
2271           __ addl (rreg_hi, lreg_hi);
2272           __ mull (rreg_lo);
2273           __ addl (lreg_hi, rreg_hi);
2274 #endif // _LP64
2275           break;
2276         default:
2277           ShouldNotReachHere();
2278       }
2279 
2280     } else if (right->is_constant()) {
2281       // cpu register - constant
2282 #ifdef _LP64
2283       jlong c = right->as_constant_ptr()->as_jlong_bits();
2284       __ movptr(r10, (intptr_t) c);
2285       switch (code) {
2286         case lir_add:
2287           __ addptr(lreg_lo, r10);
2288           break;
2289         case lir_sub:
2290           __ subptr(lreg_lo, r10);
2291           break;
2292         default:
2293           ShouldNotReachHere();
2294       }
2295 #else
2296       jint c_lo = right->as_constant_ptr()->as_jint_lo();
2297       jint c_hi = right->as_constant_ptr()->as_jint_hi();
2298       switch (code) {
2299         case lir_add:
2300           __ addptr(lreg_lo, c_lo);
2301           __ adcl(lreg_hi, c_hi);
2302           break;
2303         case lir_sub:
2304           __ subptr(lreg_lo, c_lo);
2305           __ sbbl(lreg_hi, c_hi);
2306           break;
2307         default:
2308           ShouldNotReachHere();
2309       }
2310 #endif // _LP64
2311 
2312     } else {
2313       ShouldNotReachHere();
2314     }
2315 
2316   } else if (left->is_single_xmm()) {
2317     assert(left == dest, "left and dest must be equal");
2318     XMMRegister lreg = left->as_xmm_float_reg();
2319 
2320     if (right->is_single_xmm()) {
2321       XMMRegister rreg = right->as_xmm_float_reg();
2322       switch (code) {
2323         case lir_add: __ addss(lreg, rreg);  break;
2324         case lir_sub: __ subss(lreg, rreg);  break;
2325         case lir_mul_strictfp: // fall through
2326         case lir_mul: __ mulss(lreg, rreg);  break;
2327         case lir_div_strictfp: // fall through
2328         case lir_div: __ divss(lreg, rreg);  break;
2329         default: ShouldNotReachHere();
2330       }
2331     } else {
2332       Address raddr;
2333       if (right->is_single_stack()) {
2334         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2335       } else if (right->is_constant()) {
2336         // hack for now
2337         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2338       } else {
2339         ShouldNotReachHere();
2340       }
2341       switch (code) {
2342         case lir_add: __ addss(lreg, raddr);  break;
2343         case lir_sub: __ subss(lreg, raddr);  break;
2344         case lir_mul_strictfp: // fall through
2345         case lir_mul: __ mulss(lreg, raddr);  break;
2346         case lir_div_strictfp: // fall through
2347         case lir_div: __ divss(lreg, raddr);  break;
2348         default: ShouldNotReachHere();
2349       }
2350     }
2351 
2352   } else if (left->is_double_xmm()) {
2353     assert(left == dest, "left and dest must be equal");
2354 
2355     XMMRegister lreg = left->as_xmm_double_reg();
2356     if (right->is_double_xmm()) {
2357       XMMRegister rreg = right->as_xmm_double_reg();
2358       switch (code) {
2359         case lir_add: __ addsd(lreg, rreg);  break;
2360         case lir_sub: __ subsd(lreg, rreg);  break;
2361         case lir_mul_strictfp: // fall through
2362         case lir_mul: __ mulsd(lreg, rreg);  break;
2363         case lir_div_strictfp: // fall through
2364         case lir_div: __ divsd(lreg, rreg);  break;
2365         default: ShouldNotReachHere();
2366       }
2367     } else {
2368       Address raddr;
2369       if (right->is_double_stack()) {
2370         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2371       } else if (right->is_constant()) {
2372         // hack for now
2373         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2374       } else {
2375         ShouldNotReachHere();
2376       }
2377       switch (code) {
2378         case lir_add: __ addsd(lreg, raddr);  break;
2379         case lir_sub: __ subsd(lreg, raddr);  break;
2380         case lir_mul_strictfp: // fall through
2381         case lir_mul: __ mulsd(lreg, raddr);  break;
2382         case lir_div_strictfp: // fall through
2383         case lir_div: __ divsd(lreg, raddr);  break;
2384         default: ShouldNotReachHere();
2385       }
2386     }
2387 
2388   } else if (left->is_single_fpu()) {
2389     assert(dest->is_single_fpu(),  "fpu stack allocation required");
2390 
2391     if (right->is_single_fpu()) {
2392       arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack);
2393 
2394     } else {
2395       assert(left->fpu_regnr() == 0, "left must be on TOS");
2396       assert(dest->fpu_regnr() == 0, "dest must be on TOS");
2397 
2398       Address raddr;
2399       if (right->is_single_stack()) {
2400         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2401       } else if (right->is_constant()) {
2402         address const_addr = float_constant(right->as_jfloat());
2403         assert(const_addr != NULL, "incorrect float/double constant maintainance");
2404         // hack for now
2405         raddr = __ as_Address(InternalAddress(const_addr));
2406       } else {
2407         ShouldNotReachHere();
2408       }
2409 
2410       switch (code) {
2411         case lir_add: __ fadd_s(raddr); break;
2412         case lir_sub: __ fsub_s(raddr); break;
2413         case lir_mul_strictfp: // fall through
2414         case lir_mul: __ fmul_s(raddr); break;
2415         case lir_div_strictfp: // fall through
2416         case lir_div: __ fdiv_s(raddr); break;
2417         default:      ShouldNotReachHere();
2418       }
2419     }
2420 
2421   } else if (left->is_double_fpu()) {
2422     assert(dest->is_double_fpu(),  "fpu stack allocation required");
2423 
2424     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2425       // Double values require special handling for strictfp mul/div on x86
2426       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
2427       __ fmulp(left->fpu_regnrLo() + 1);
2428     }
2429 
2430     if (right->is_double_fpu()) {
2431       arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack);
2432 
2433     } else {
2434       assert(left->fpu_regnrLo() == 0, "left must be on TOS");
2435       assert(dest->fpu_regnrLo() == 0, "dest must be on TOS");
2436 
2437       Address raddr;
2438       if (right->is_double_stack()) {
2439         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2440       } else if (right->is_constant()) {
2441         // hack for now
2442         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2443       } else {
2444         ShouldNotReachHere();
2445       }
2446 
2447       switch (code) {
2448         case lir_add: __ fadd_d(raddr); break;
2449         case lir_sub: __ fsub_d(raddr); break;
2450         case lir_mul_strictfp: // fall through
2451         case lir_mul: __ fmul_d(raddr); break;
2452         case lir_div_strictfp: // fall through
2453         case lir_div: __ fdiv_d(raddr); break;
2454         default: ShouldNotReachHere();
2455       }
2456     }
2457 
2458     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2459       // Double values require special handling for strictfp mul/div on x86
2460       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
2461       __ fmulp(dest->fpu_regnrLo() + 1);
2462     }
2463 
2464   } else if (left->is_single_stack() || left->is_address()) {
2465     assert(left == dest, "left and dest must be equal");
2466 
2467     Address laddr;
2468     if (left->is_single_stack()) {
2469       laddr = frame_map()->address_for_slot(left->single_stack_ix());
2470     } else if (left->is_address()) {
2471       laddr = as_Address(left->as_address_ptr());
2472     } else {
2473       ShouldNotReachHere();
2474     }
2475 
2476     if (right->is_single_cpu()) {
2477       Register rreg = right->as_register();
2478       switch (code) {
2479         case lir_add: __ addl(laddr, rreg); break;
2480         case lir_sub: __ subl(laddr, rreg); break;
2481         default:      ShouldNotReachHere();
2482       }
2483     } else if (right->is_constant()) {
2484       jint c = right->as_constant_ptr()->as_jint();
2485       switch (code) {
2486         case lir_add: {
2487           __ incrementl(laddr, c);
2488           break;
2489         }
2490         case lir_sub: {
2491           __ decrementl(laddr, c);
2492           break;
2493         }
2494         default: ShouldNotReachHere();
2495       }
2496     } else {
2497       ShouldNotReachHere();
2498     }
2499 
2500   } else {
2501     ShouldNotReachHere();
2502   }
2503 }
2504 
2505 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) {
2506   assert(pop_fpu_stack  || (left_index     == dest_index || right_index     == dest_index), "invalid LIR");
2507   assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR");
2508   assert(left_index == 0 || right_index == 0, "either must be on top of stack");
2509 
2510   bool left_is_tos = (left_index == 0);
2511   bool dest_is_tos = (dest_index == 0);
2512   int non_tos_index = (left_is_tos ? right_index : left_index);
2513 
2514   switch (code) {
2515     case lir_add:
2516       if (pop_fpu_stack)       __ faddp(non_tos_index);
2517       else if (dest_is_tos)    __ fadd (non_tos_index);
2518       else                     __ fadda(non_tos_index);
2519       break;
2520 
2521     case lir_sub:
2522       if (left_is_tos) {
2523         if (pop_fpu_stack)     __ fsubrp(non_tos_index);
2524         else if (dest_is_tos)  __ fsub  (non_tos_index);
2525         else                   __ fsubra(non_tos_index);
2526       } else {
2527         if (pop_fpu_stack)     __ fsubp (non_tos_index);
2528         else if (dest_is_tos)  __ fsubr (non_tos_index);
2529         else                   __ fsuba (non_tos_index);
2530       }
2531       break;
2532 
2533     case lir_mul_strictfp: // fall through
2534     case lir_mul:
2535       if (pop_fpu_stack)       __ fmulp(non_tos_index);
2536       else if (dest_is_tos)    __ fmul (non_tos_index);
2537       else                     __ fmula(non_tos_index);
2538       break;
2539 
2540     case lir_div_strictfp: // fall through
2541     case lir_div:
2542       if (left_is_tos) {
2543         if (pop_fpu_stack)     __ fdivrp(non_tos_index);
2544         else if (dest_is_tos)  __ fdiv  (non_tos_index);
2545         else                   __ fdivra(non_tos_index);
2546       } else {
2547         if (pop_fpu_stack)     __ fdivp (non_tos_index);
2548         else if (dest_is_tos)  __ fdivr (non_tos_index);
2549         else                   __ fdiva (non_tos_index);
2550       }
2551       break;
2552 
2553     case lir_rem:
2554       assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation");
2555       __ fremr(noreg);
2556       break;
2557 
2558     default:
2559       ShouldNotReachHere();
2560   }
2561 }
2562 
2563 
2564 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2565   if (value->is_double_xmm()) {
2566     switch(code) {
2567       case lir_abs :
2568         {
2569 #ifdef _LP64
2570           if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2571             assert(tmp->is_valid(), "need temporary");
2572             __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2573           } else
2574 #endif
2575           {
2576             if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2577               __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2578             }
2579             assert(!tmp->is_valid(), "do not need temporary");
2580             __ andpd(dest->as_xmm_double_reg(),
2581                      ExternalAddress((address)double_signmask_pool));
2582           }
2583         }
2584         break;
2585 
2586       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2587       // all other intrinsics are not available in the SSE instruction set, so FPU is used
2588       default      : ShouldNotReachHere();
2589     }
2590 
2591   } else if (value->is_double_fpu()) {
2592     assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2593     switch(code) {
2594       case lir_abs   : __ fabs() ; break;
2595       case lir_sqrt  : __ fsqrt(); break;
2596       default      : ShouldNotReachHere();
2597     }
2598   } else {
2599     Unimplemented();
2600   }
2601 }
2602 
2603 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2604   // assert(left->destroys_register(), "check");
2605   if (left->is_single_cpu()) {
2606     Register reg = left->as_register();
2607     if (right->is_constant()) {
2608       int val = right->as_constant_ptr()->as_jint();
2609       switch (code) {
2610         case lir_logic_and: __ andl (reg, val); break;
2611         case lir_logic_or:  __ orl  (reg, val); break;
2612         case lir_logic_xor: __ xorl (reg, val); break;
2613         default: ShouldNotReachHere();
2614       }
2615     } else if (right->is_stack()) {
2616       // added support for stack operands
2617       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2618       switch (code) {
2619         case lir_logic_and: __ andl (reg, raddr); break;
2620         case lir_logic_or:  __ orl  (reg, raddr); break;
2621         case lir_logic_xor: __ xorl (reg, raddr); break;
2622         default: ShouldNotReachHere();
2623       }
2624     } else {
2625       Register rright = right->as_register();
2626       switch (code) {
2627         case lir_logic_and: __ andptr (reg, rright); break;
2628         case lir_logic_or : __ orptr  (reg, rright); break;
2629         case lir_logic_xor: __ xorptr (reg, rright); break;
2630         default: ShouldNotReachHere();
2631       }
2632     }
2633     move_regs(reg, dst->as_register());
2634   } else {
2635     Register l_lo = left->as_register_lo();
2636     Register l_hi = left->as_register_hi();
2637     if (right->is_constant()) {
2638 #ifdef _LP64
2639       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2640       switch (code) {
2641         case lir_logic_and:
2642           __ andq(l_lo, rscratch1);
2643           break;
2644         case lir_logic_or:
2645           __ orq(l_lo, rscratch1);
2646           break;
2647         case lir_logic_xor:
2648           __ xorq(l_lo, rscratch1);
2649           break;
2650         default: ShouldNotReachHere();
2651       }
2652 #else
2653       int r_lo = right->as_constant_ptr()->as_jint_lo();
2654       int r_hi = right->as_constant_ptr()->as_jint_hi();
2655       switch (code) {
2656         case lir_logic_and:
2657           __ andl(l_lo, r_lo);
2658           __ andl(l_hi, r_hi);
2659           break;
2660         case lir_logic_or:
2661           __ orl(l_lo, r_lo);
2662           __ orl(l_hi, r_hi);
2663           break;
2664         case lir_logic_xor:
2665           __ xorl(l_lo, r_lo);
2666           __ xorl(l_hi, r_hi);
2667           break;
2668         default: ShouldNotReachHere();
2669       }
2670 #endif // _LP64
2671     } else {
2672 #ifdef _LP64
2673       Register r_lo;
2674       if (right->type() == T_OBJECT || right->type() == T_ARRAY || right->type() == T_VALUETYPE) {
2675         r_lo = right->as_register();
2676       } else {
2677         r_lo = right->as_register_lo();
2678       }
2679 #else
2680       Register r_lo = right->as_register_lo();
2681       Register r_hi = right->as_register_hi();
2682       assert(l_lo != r_hi, "overwriting registers");
2683 #endif
2684       switch (code) {
2685         case lir_logic_and:
2686           __ andptr(l_lo, r_lo);
2687           NOT_LP64(__ andptr(l_hi, r_hi);)
2688           break;
2689         case lir_logic_or:
2690           __ orptr(l_lo, r_lo);
2691           NOT_LP64(__ orptr(l_hi, r_hi);)
2692           break;
2693         case lir_logic_xor:
2694           __ xorptr(l_lo, r_lo);
2695           NOT_LP64(__ xorptr(l_hi, r_hi);)
2696           break;
2697         default: ShouldNotReachHere();
2698       }
2699     }
2700 
2701     Register dst_lo = dst->as_register_lo();
2702     Register dst_hi = dst->as_register_hi();
2703 
2704 #ifdef _LP64
2705     move_regs(l_lo, dst_lo);
2706 #else
2707     if (dst_lo == l_hi) {
2708       assert(dst_hi != l_lo, "overwriting registers");
2709       move_regs(l_hi, dst_hi);
2710       move_regs(l_lo, dst_lo);
2711     } else {
2712       assert(dst_lo != l_hi, "overwriting registers");
2713       move_regs(l_lo, dst_lo);
2714       move_regs(l_hi, dst_hi);
2715     }
2716 #endif // _LP64
2717   }
2718 }
2719 
2720 
2721 // we assume that rax, and rdx can be overwritten
2722 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2723 
2724   assert(left->is_single_cpu(),   "left must be register");
2725   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2726   assert(result->is_single_cpu(), "result must be register");
2727 
2728   //  assert(left->destroys_register(), "check");
2729   //  assert(right->destroys_register(), "check");
2730 
2731   Register lreg = left->as_register();
2732   Register dreg = result->as_register();
2733 
2734   if (right->is_constant()) {
2735     jint divisor = right->as_constant_ptr()->as_jint();
2736     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2737     if (code == lir_idiv) {
2738       assert(lreg == rax, "must be rax,");
2739       assert(temp->as_register() == rdx, "tmp register must be rdx");
2740       __ cdql(); // sign extend into rdx:rax
2741       if (divisor == 2) {
2742         __ subl(lreg, rdx);
2743       } else {
2744         __ andl(rdx, divisor - 1);
2745         __ addl(lreg, rdx);
2746       }
2747       __ sarl(lreg, log2_jint(divisor));
2748       move_regs(lreg, dreg);
2749     } else if (code == lir_irem) {
2750       Label done;
2751       __ mov(dreg, lreg);
2752       __ andl(dreg, 0x80000000 | (divisor - 1));
2753       __ jcc(Assembler::positive, done);
2754       __ decrement(dreg);
2755       __ orl(dreg, ~(divisor - 1));
2756       __ increment(dreg);
2757       __ bind(done);
2758     } else {
2759       ShouldNotReachHere();
2760     }
2761   } else {
2762     Register rreg = right->as_register();
2763     assert(lreg == rax, "left register must be rax,");
2764     assert(rreg != rdx, "right register must not be rdx");
2765     assert(temp->as_register() == rdx, "tmp register must be rdx");
2766 
2767     move_regs(lreg, rax);
2768 
2769     int idivl_offset = __ corrected_idivl(rreg);
2770     if (ImplicitDiv0Checks) {
2771       add_debug_info_for_div0(idivl_offset, info);
2772     }
2773     if (code == lir_irem) {
2774       move_regs(rdx, dreg); // result is in rdx
2775     } else {
2776       move_regs(rax, dreg);
2777     }
2778   }
2779 }
2780 
2781 
2782 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2783   if (opr1->is_single_cpu()) {
2784     Register reg1 = opr1->as_register();
2785     if (opr2->is_single_cpu()) {
2786       // cpu register - cpu register
2787       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_VALUETYPE) {
2788         __ cmpoop(reg1, opr2->as_register());
2789       } else {
2790         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY && opr2->type() != T_VALUETYPE, "cmp int, oop?");
2791         __ cmpl(reg1, opr2->as_register());
2792       }
2793     } else if (opr2->is_stack()) {
2794       // cpu register - stack
2795       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_VALUETYPE) {
2796         __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2797       } else {
2798         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2799       }
2800     } else if (opr2->is_constant()) {
2801       // cpu register - constant
2802       LIR_Const* c = opr2->as_constant_ptr();
2803       if (c->type() == T_INT) {
2804         __ cmpl(reg1, c->as_jint());
2805       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY || c->type() == T_VALUETYPE) {
2806         // In 64bit oops are single register
2807         jobject o = c->as_jobject();
2808         if (o == NULL) {
2809           __ cmpptr(reg1, (int32_t)NULL_WORD);
2810         } else {
2811           __ cmpoop(reg1, o);
2812         }
2813       } else {
2814         fatal("unexpected type: %s", basictype_to_str(c->type()));
2815       }
2816       // cpu register - address
2817     } else if (opr2->is_address()) {
2818       if (op->info() != NULL) {
2819         add_debug_info_for_null_check_here(op->info());
2820       }
2821       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2822     } else {
2823       ShouldNotReachHere();
2824     }
2825 
2826   } else if(opr1->is_double_cpu()) {
2827     Register xlo = opr1->as_register_lo();
2828     Register xhi = opr1->as_register_hi();
2829     if (opr2->is_double_cpu()) {
2830 #ifdef _LP64
2831       __ cmpptr(xlo, opr2->as_register_lo());
2832 #else
2833       // cpu register - cpu register
2834       Register ylo = opr2->as_register_lo();
2835       Register yhi = opr2->as_register_hi();
2836       __ subl(xlo, ylo);
2837       __ sbbl(xhi, yhi);
2838       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2839         __ orl(xhi, xlo);
2840       }
2841 #endif // _LP64
2842     } else if (opr2->is_constant()) {
2843       // cpu register - constant 0
2844       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2845 #ifdef _LP64
2846       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2847 #else
2848       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2849       __ orl(xhi, xlo);
2850 #endif // _LP64
2851     } else {
2852       ShouldNotReachHere();
2853     }
2854 
2855   } else if (opr1->is_single_xmm()) {
2856     XMMRegister reg1 = opr1->as_xmm_float_reg();
2857     if (opr2->is_single_xmm()) {
2858       // xmm register - xmm register
2859       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2860     } else if (opr2->is_stack()) {
2861       // xmm register - stack
2862       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2863     } else if (opr2->is_constant()) {
2864       // xmm register - constant
2865       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2866     } else if (opr2->is_address()) {
2867       // xmm register - address
2868       if (op->info() != NULL) {
2869         add_debug_info_for_null_check_here(op->info());
2870       }
2871       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2872     } else {
2873       ShouldNotReachHere();
2874     }
2875 
2876   } else if (opr1->is_double_xmm()) {
2877     XMMRegister reg1 = opr1->as_xmm_double_reg();
2878     if (opr2->is_double_xmm()) {
2879       // xmm register - xmm register
2880       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2881     } else if (opr2->is_stack()) {
2882       // xmm register - stack
2883       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2884     } else if (opr2->is_constant()) {
2885       // xmm register - constant
2886       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2887     } else if (opr2->is_address()) {
2888       // xmm register - address
2889       if (op->info() != NULL) {
2890         add_debug_info_for_null_check_here(op->info());
2891       }
2892       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2893     } else {
2894       ShouldNotReachHere();
2895     }
2896 
2897   } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) {
2898     assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)");
2899     assert(opr2->is_fpu_register(), "both must be registers");
2900     __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2901 
2902   } else if (opr1->is_address() && opr2->is_constant()) {
2903     LIR_Const* c = opr2->as_constant_ptr();
2904 #ifdef _LP64
2905     if (c->type() == T_OBJECT || c->type() == T_ARRAY || c->type() == T_VALUETYPE) {
2906       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2907       __ movoop(rscratch1, c->as_jobject());
2908     }
2909 #endif // LP64
2910     if (op->info() != NULL) {
2911       add_debug_info_for_null_check_here(op->info());
2912     }
2913     // special case: address - constant
2914     LIR_Address* addr = opr1->as_address_ptr();
2915     if (c->type() == T_INT) {
2916       __ cmpl(as_Address(addr), c->as_jint());
2917     } else if (c->type() == T_OBJECT || c->type() == T_ARRAY || c->type() == T_VALUETYPE) {
2918 #ifdef _LP64
2919       // %%% Make this explode if addr isn't reachable until we figure out a
2920       // better strategy by giving noreg as the temp for as_Address
2921       __ cmpoop(rscratch1, as_Address(addr, noreg));
2922 #else
2923       __ cmpoop(as_Address(addr), c->as_jobject());
2924 #endif // _LP64
2925     } else {
2926       ShouldNotReachHere();
2927     }
2928 
2929   } else {
2930     ShouldNotReachHere();
2931   }
2932 }
2933 
2934 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2935   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2936     if (left->is_single_xmm()) {
2937       assert(right->is_single_xmm(), "must match");
2938       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2939     } else if (left->is_double_xmm()) {
2940       assert(right->is_double_xmm(), "must match");
2941       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2942 
2943     } else {
2944       assert(left->is_single_fpu() || left->is_double_fpu(), "must be");
2945       assert(right->is_single_fpu() || right->is_double_fpu(), "must match");
2946 
2947       assert(left->fpu() == 0, "left must be on TOS");
2948       __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2949                   op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2950     }
2951   } else {
2952     assert(code == lir_cmp_l2i, "check");
2953 #ifdef _LP64
2954     Label done;
2955     Register dest = dst->as_register();
2956     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2957     __ movl(dest, -1);
2958     __ jccb(Assembler::less, done);
2959     __ set_byte_if_not_zero(dest);
2960     __ movzbl(dest, dest);
2961     __ bind(done);
2962 #else
2963     __ lcmp2int(left->as_register_hi(),
2964                 left->as_register_lo(),
2965                 right->as_register_hi(),
2966                 right->as_register_lo());
2967     move_regs(left->as_register_hi(), dst->as_register());
2968 #endif // _LP64
2969   }
2970 }
2971 
2972 
2973 void LIR_Assembler::align_call(LIR_Code code) {
2974   // make sure that the displacement word of the call ends up word aligned
2975   int offset = __ offset();
2976   switch (code) {
2977   case lir_static_call:
2978   case lir_optvirtual_call:
2979   case lir_dynamic_call:
2980     offset += NativeCall::displacement_offset;
2981     break;
2982   case lir_icvirtual_call:
2983     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
2984     break;
2985   case lir_virtual_call:  // currently, sparc-specific for niagara
2986   default: ShouldNotReachHere();
2987   }
2988   __ align(BytesPerWord, offset);
2989 }
2990 
2991 
2992 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2993   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2994          "must be aligned");
2995   __ call(AddressLiteral(op->addr(), rtype));
2996   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2997 }
2998 
2999 
3000 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
3001   __ ic_call(op->addr());
3002   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3003   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
3004          "must be aligned");
3005 }
3006 
3007 
3008 /* Currently, vtable-dispatch is only enabled for sparc platforms */
3009 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
3010   ShouldNotReachHere();
3011 }
3012 
3013 
3014 void LIR_Assembler::emit_static_call_stub() {
3015   address call_pc = __ pc();
3016   address stub = __ start_a_stub(call_stub_size());
3017   if (stub == NULL) {
3018     bailout("static call stub overflow");
3019     return;
3020   }
3021 
3022   int start = __ offset();
3023 
3024   // make sure that the displacement word of the call ends up word aligned
3025   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset);
3026   __ relocate(static_stub_Relocation::spec(call_pc, false /* is_aot */));
3027   __ mov_metadata(rbx, (Metadata*)NULL);
3028   // must be set to -1 at code generation time
3029   assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
3030   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
3031   __ jump(RuntimeAddress(__ pc()));
3032 
3033   if (UseAOT) {
3034     // Trampoline to aot code
3035     __ relocate(static_stub_Relocation::spec(call_pc, true /* is_aot */));
3036 #ifdef _LP64
3037     __ mov64(rax, CONST64(0));  // address is zapped till fixup time.
3038 #else
3039     __ movl(rax, 0xdeadffff);  // address is zapped till fixup time.
3040 #endif
3041     __ jmp(rax);
3042   }
3043   assert(__ offset() - start <= call_stub_size(), "stub too big");
3044   __ end_a_stub();
3045 }
3046 
3047 
3048 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
3049   assert(exceptionOop->as_register() == rax, "must match");
3050   assert(exceptionPC->as_register() == rdx, "must match");
3051 
3052   // exception object is not added to oop map by LinearScan
3053   // (LinearScan assumes that no oops are in fixed registers)
3054   info->add_register_oop(exceptionOop);
3055   Runtime1::StubID unwind_id;
3056 
3057   // get current pc information
3058   // pc is only needed if the method has an exception handler, the unwind code does not need it.
3059   int pc_for_athrow_offset = __ offset();
3060   InternalAddress pc_for_athrow(__ pc());
3061   __ lea(exceptionPC->as_register(), pc_for_athrow);
3062   add_call_info(pc_for_athrow_offset, info); // for exception handler
3063 
3064   __ verify_not_null_oop(rax);
3065   // search an exception handler (rax: exception oop, rdx: throwing pc)
3066   if (compilation()->has_fpu_code()) {
3067     unwind_id = Runtime1::handle_exception_id;
3068   } else {
3069     unwind_id = Runtime1::handle_exception_nofpu_id;
3070   }
3071   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
3072 
3073   // enough room for two byte trap
3074   __ nop();
3075 }
3076 
3077 
3078 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
3079   assert(exceptionOop->as_register() == rax, "must match");
3080 
3081   __ jmp(_unwind_handler_entry);
3082 }
3083 
3084 
3085 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
3086 
3087   // optimized version for linear scan:
3088   // * count must be already in ECX (guaranteed by LinearScan)
3089   // * left and dest must be equal
3090   // * tmp must be unused
3091   assert(count->as_register() == SHIFT_count, "count must be in ECX");
3092   assert(left == dest, "left and dest must be equal");
3093   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3094 
3095   if (left->is_single_cpu()) {
3096     Register value = left->as_register();
3097     assert(value != SHIFT_count, "left cannot be ECX");
3098 
3099     switch (code) {
3100       case lir_shl:  __ shll(value); break;
3101       case lir_shr:  __ sarl(value); break;
3102       case lir_ushr: __ shrl(value); break;
3103       default: ShouldNotReachHere();
3104     }
3105   } else if (left->is_double_cpu()) {
3106     Register lo = left->as_register_lo();
3107     Register hi = left->as_register_hi();
3108     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
3109 #ifdef _LP64
3110     switch (code) {
3111       case lir_shl:  __ shlptr(lo);        break;
3112       case lir_shr:  __ sarptr(lo);        break;
3113       case lir_ushr: __ shrptr(lo);        break;
3114       default: ShouldNotReachHere();
3115     }
3116 #else
3117 
3118     switch (code) {
3119       case lir_shl:  __ lshl(hi, lo);        break;
3120       case lir_shr:  __ lshr(hi, lo, true);  break;
3121       case lir_ushr: __ lshr(hi, lo, false); break;
3122       default: ShouldNotReachHere();
3123     }
3124 #endif // LP64
3125   } else {
3126     ShouldNotReachHere();
3127   }
3128 }
3129 
3130 
3131 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
3132   if (dest->is_single_cpu()) {
3133     // first move left into dest so that left is not destroyed by the shift
3134     Register value = dest->as_register();
3135     count = count & 0x1F; // Java spec
3136 
3137     move_regs(left->as_register(), value);
3138     switch (code) {
3139       case lir_shl:  __ shll(value, count); break;
3140       case lir_shr:  __ sarl(value, count); break;
3141       case lir_ushr: __ shrl(value, count); break;
3142       default: ShouldNotReachHere();
3143     }
3144   } else if (dest->is_double_cpu()) {
3145 #ifndef _LP64
3146     Unimplemented();
3147 #else
3148     // first move left into dest so that left is not destroyed by the shift
3149     Register value = dest->as_register_lo();
3150     count = count & 0x1F; // Java spec
3151 
3152     move_regs(left->as_register_lo(), value);
3153     switch (code) {
3154       case lir_shl:  __ shlptr(value, count); break;
3155       case lir_shr:  __ sarptr(value, count); break;
3156       case lir_ushr: __ shrptr(value, count); break;
3157       default: ShouldNotReachHere();
3158     }
3159 #endif // _LP64
3160   } else {
3161     ShouldNotReachHere();
3162   }
3163 }
3164 
3165 
3166 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
3167   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3168   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3169   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3170   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
3171 }
3172 
3173 
3174 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
3175   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3176   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3177   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3178   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3179 }
3180 
3181 
3182 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
3183   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3184   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3185   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3186   __ movoop (Address(rsp, offset_from_rsp_in_bytes), o);
3187 }
3188 
3189 
3190 void LIR_Assembler::store_parameter(Metadata* m,  int offset_from_rsp_in_words) {
3191   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3192   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3193   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3194   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m);
3195 }
3196 
3197 
3198 void LIR_Assembler::arraycopy_valuetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
3199   if (null_check) {
3200     __ testptr(obj, obj);
3201     __ jcc(Assembler::zero, *slow_path->entry());
3202   }
3203   __ load_storage_props(tmp, obj);
3204   if (is_dest) {
3205     // We also take slow path if it's a null_free destination array, just in case the source array
3206     // contains NULLs.
3207     __ testb(tmp, ArrayStorageProperties::flattened_value | ArrayStorageProperties::null_free_value);
3208   } else {
3209     __ testb(tmp, ArrayStorageProperties::flattened_value);
3210   }
3211   __ jcc(Assembler::notEqual, *slow_path->entry());
3212 }
3213 
3214 
3215 // This code replaces a call to arraycopy; no exception may
3216 // be thrown in this code, they must be thrown in the System.arraycopy
3217 // activation frame; we could save some checks if this would not be the case
3218 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3219   ciArrayKlass* default_type = op->expected_type();
3220   Register src = op->src()->as_register();
3221   Register dst = op->dst()->as_register();
3222   Register src_pos = op->src_pos()->as_register();
3223   Register dst_pos = op->dst_pos()->as_register();
3224   Register length  = op->length()->as_register();
3225   Register tmp = op->tmp()->as_register();
3226 
3227   __ resolve(ACCESS_READ, src);
3228   __ resolve(ACCESS_WRITE, dst);
3229 
3230   CodeStub* stub = op->stub();
3231   int flags = op->flags();
3232   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
3233   if (basic_type == T_ARRAY || basic_type == T_VALUETYPE) basic_type = T_OBJECT;
3234 
3235   if (flags & LIR_OpArrayCopy::always_slow_path) {
3236     __ jmp(*stub->entry());
3237     __ bind(*stub->continuation());
3238     return;
3239   }
3240 
3241   if (flags & LIR_OpArrayCopy::src_valuetype_check) {
3242     arraycopy_valuetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
3243   }
3244 
3245   if (flags & LIR_OpArrayCopy::dst_valuetype_check) {
3246     arraycopy_valuetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
3247   }
3248 
3249   // if we don't know anything, just go through the generic arraycopy
3250   if (default_type == NULL) {
3251     // save outgoing arguments on stack in case call to System.arraycopy is needed
3252     // HACK ALERT. This code used to push the parameters in a hardwired fashion
3253     // for interpreter calling conventions. Now we have to do it in new style conventions.
3254     // For the moment until C1 gets the new register allocator I just force all the
3255     // args to the right place (except the register args) and then on the back side
3256     // reload the register args properly if we go slow path. Yuck
3257 
3258     // These are proper for the calling convention
3259     store_parameter(length, 2);
3260     store_parameter(dst_pos, 1);
3261     store_parameter(dst, 0);
3262 
3263     // these are just temporary placements until we need to reload
3264     store_parameter(src_pos, 3);
3265     store_parameter(src, 4);
3266     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3267 
3268     address copyfunc_addr = StubRoutines::generic_arraycopy();
3269     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
3270 
3271     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
3272 #ifdef _LP64
3273     // The arguments are in java calling convention so we can trivially shift them to C
3274     // convention
3275     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
3276     __ mov(c_rarg0, j_rarg0);
3277     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
3278     __ mov(c_rarg1, j_rarg1);
3279     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
3280     __ mov(c_rarg2, j_rarg2);
3281     assert_different_registers(c_rarg3, j_rarg4);
3282     __ mov(c_rarg3, j_rarg3);
3283 #ifdef _WIN64
3284     // Allocate abi space for args but be sure to keep stack aligned
3285     __ subptr(rsp, 6*wordSize);
3286     store_parameter(j_rarg4, 4);
3287 #ifndef PRODUCT
3288     if (PrintC1Statistics) {
3289       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3290     }
3291 #endif
3292     __ call(RuntimeAddress(copyfunc_addr));
3293     __ addptr(rsp, 6*wordSize);
3294 #else
3295     __ mov(c_rarg4, j_rarg4);
3296 #ifndef PRODUCT
3297     if (PrintC1Statistics) {
3298       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3299     }
3300 #endif
3301     __ call(RuntimeAddress(copyfunc_addr));
3302 #endif // _WIN64
3303 #else
3304     __ push(length);
3305     __ push(dst_pos);
3306     __ push(dst);
3307     __ push(src_pos);
3308     __ push(src);
3309 
3310 #ifndef PRODUCT
3311     if (PrintC1Statistics) {
3312       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3313     }
3314 #endif
3315     __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
3316 
3317 #endif // _LP64
3318 
3319     __ cmpl(rax, 0);
3320     __ jcc(Assembler::equal, *stub->continuation());
3321 
3322     __ mov(tmp, rax);
3323     __ xorl(tmp, -1);
3324 
3325     // Reload values from the stack so they are where the stub
3326     // expects them.
3327     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3328     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3329     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3330     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3331     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3332 
3333     __ subl(length, tmp);
3334     __ addl(src_pos, tmp);
3335     __ addl(dst_pos, tmp);
3336     __ jmp(*stub->entry());
3337 
3338     __ bind(*stub->continuation());
3339     return;
3340   }
3341 
3342   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3343 
3344   int elem_size = type2aelembytes(basic_type);
3345   Address::ScaleFactor scale;
3346 
3347   switch (elem_size) {
3348     case 1 :
3349       scale = Address::times_1;
3350       break;
3351     case 2 :
3352       scale = Address::times_2;
3353       break;
3354     case 4 :
3355       scale = Address::times_4;
3356       break;
3357     case 8 :
3358       scale = Address::times_8;
3359       break;
3360     default:
3361       scale = Address::no_scale;
3362       ShouldNotReachHere();
3363   }
3364 
3365   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3366   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3367   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3368   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3369 
3370   // length and pos's are all sign extended at this point on 64bit
3371 
3372   // test for NULL
3373   if (flags & LIR_OpArrayCopy::src_null_check) {
3374     __ testptr(src, src);
3375     __ jcc(Assembler::zero, *stub->entry());
3376   }
3377   if (flags & LIR_OpArrayCopy::dst_null_check) {
3378     __ testptr(dst, dst);
3379     __ jcc(Assembler::zero, *stub->entry());
3380   }
3381 
3382   // If the compiler was not able to prove that exact type of the source or the destination
3383   // of the arraycopy is an array type, check at runtime if the source or the destination is
3384   // an instance type.
3385   if (flags & LIR_OpArrayCopy::type_check) {
3386     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3387       __ load_klass(tmp, dst);
3388       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3389       __ jcc(Assembler::greaterEqual, *stub->entry());
3390     }
3391 
3392     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3393       __ load_klass(tmp, src);
3394       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3395       __ jcc(Assembler::greaterEqual, *stub->entry());
3396     }
3397   }
3398 
3399   // check if negative
3400   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3401     __ testl(src_pos, src_pos);
3402     __ jcc(Assembler::less, *stub->entry());
3403   }
3404   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3405     __ testl(dst_pos, dst_pos);
3406     __ jcc(Assembler::less, *stub->entry());
3407   }
3408 
3409   if (flags & LIR_OpArrayCopy::src_range_check) {
3410     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3411     __ cmpl(tmp, src_length_addr);
3412     __ jcc(Assembler::above, *stub->entry());
3413   }
3414   if (flags & LIR_OpArrayCopy::dst_range_check) {
3415     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3416     __ cmpl(tmp, dst_length_addr);
3417     __ jcc(Assembler::above, *stub->entry());
3418   }
3419 
3420   if (flags & LIR_OpArrayCopy::length_positive_check) {
3421     __ testl(length, length);
3422     __ jcc(Assembler::less, *stub->entry());
3423   }
3424 
3425 #ifdef _LP64
3426   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3427   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3428 #endif
3429 
3430   if (flags & LIR_OpArrayCopy::type_check) {
3431     // We don't know the array types are compatible
3432     if (basic_type != T_OBJECT) {
3433       // Simple test for basic type arrays
3434       if (UseCompressedClassPointers) {
3435         __ movl(tmp, src_klass_addr);
3436         __ cmpl(tmp, dst_klass_addr);
3437       } else {
3438         __ movptr(tmp, src_klass_addr);
3439         __ cmpptr(tmp, dst_klass_addr);
3440       }
3441       __ jcc(Assembler::notEqual, *stub->entry());
3442     } else {
3443       // For object arrays, if src is a sub class of dst then we can
3444       // safely do the copy.
3445       Label cont, slow;
3446 
3447       __ push(src);
3448       __ push(dst);
3449 
3450       __ load_klass(src, src);
3451       __ load_klass(dst, dst);
3452 
3453       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
3454 
3455       __ push(src);
3456       __ push(dst);
3457       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3458       __ pop(dst);
3459       __ pop(src);
3460 
3461       __ cmpl(src, 0);
3462       __ jcc(Assembler::notEqual, cont);
3463 
3464       __ bind(slow);
3465       __ pop(dst);
3466       __ pop(src);
3467 
3468       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
3469       if (copyfunc_addr != NULL) { // use stub if available
3470         // src is not a sub class of dst so we have to do a
3471         // per-element check.
3472 
3473         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
3474         if ((flags & mask) != mask) {
3475           // Check that at least both of them object arrays.
3476           assert(flags & mask, "one of the two should be known to be an object array");
3477 
3478           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3479             __ load_klass(tmp, src);
3480           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3481             __ load_klass(tmp, dst);
3482           }
3483           int lh_offset = in_bytes(Klass::layout_helper_offset());
3484           Address klass_lh_addr(tmp, lh_offset);
3485           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3486           __ cmpl(klass_lh_addr, objArray_lh);
3487           __ jcc(Assembler::notEqual, *stub->entry());
3488         }
3489 
3490        // Spill because stubs can use any register they like and it's
3491        // easier to restore just those that we care about.
3492        store_parameter(dst, 0);
3493        store_parameter(dst_pos, 1);
3494        store_parameter(length, 2);
3495        store_parameter(src_pos, 3);
3496        store_parameter(src, 4);
3497 
3498 #ifndef _LP64
3499         __ movptr(tmp, dst_klass_addr);
3500         __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
3501         __ push(tmp);
3502         __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3503         __ push(tmp);
3504         __ push(length);
3505         __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3506         __ push(tmp);
3507         __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3508         __ push(tmp);
3509 
3510         __ call_VM_leaf(copyfunc_addr, 5);
3511 #else
3512         __ movl2ptr(length, length); //higher 32bits must be null
3513 
3514         __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3515         assert_different_registers(c_rarg0, dst, dst_pos, length);
3516         __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3517         assert_different_registers(c_rarg1, dst, length);
3518 
3519         __ mov(c_rarg2, length);
3520         assert_different_registers(c_rarg2, dst);
3521 
3522 #ifdef _WIN64
3523         // Allocate abi space for args but be sure to keep stack aligned
3524         __ subptr(rsp, 6*wordSize);
3525         __ load_klass(c_rarg3, dst);
3526         __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
3527         store_parameter(c_rarg3, 4);
3528         __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
3529         __ call(RuntimeAddress(copyfunc_addr));
3530         __ addptr(rsp, 6*wordSize);
3531 #else
3532         __ load_klass(c_rarg4, dst);
3533         __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
3534         __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
3535         __ call(RuntimeAddress(copyfunc_addr));
3536 #endif
3537 
3538 #endif
3539 
3540 #ifndef PRODUCT
3541         if (PrintC1Statistics) {
3542           Label failed;
3543           __ testl(rax, rax);
3544           __ jcc(Assembler::notZero, failed);
3545           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
3546           __ bind(failed);
3547         }
3548 #endif
3549 
3550         __ testl(rax, rax);
3551         __ jcc(Assembler::zero, *stub->continuation());
3552 
3553 #ifndef PRODUCT
3554         if (PrintC1Statistics) {
3555           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
3556         }
3557 #endif
3558 
3559         __ mov(tmp, rax);
3560 
3561         __ xorl(tmp, -1);
3562 
3563         // Restore previously spilled arguments
3564         __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3565         __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3566         __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3567         __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3568         __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3569 
3570 
3571         __ subl(length, tmp);
3572         __ addl(src_pos, tmp);
3573         __ addl(dst_pos, tmp);
3574       }
3575 
3576       __ jmp(*stub->entry());
3577 
3578       __ bind(cont);
3579       __ pop(dst);
3580       __ pop(src);
3581     }
3582   }
3583 
3584 #ifdef ASSERT
3585   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3586     // Sanity check the known type with the incoming class.  For the
3587     // primitive case the types must match exactly with src.klass and
3588     // dst.klass each exactly matching the default type.  For the
3589     // object array case, if no type check is needed then either the
3590     // dst type is exactly the expected type and the src type is a
3591     // subtype which we can't check or src is the same array as dst
3592     // but not necessarily exactly of type default_type.
3593     Label known_ok, halt;
3594     __ mov_metadata(tmp, default_type->constant_encoding());
3595 #ifdef _LP64
3596     if (UseCompressedClassPointers) {
3597       __ encode_klass_not_null(tmp);
3598     }
3599 #endif
3600 
3601     if (basic_type != T_OBJECT) {
3602 
3603       if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3604       else                   __ cmpptr(tmp, dst_klass_addr);
3605       __ jcc(Assembler::notEqual, halt);
3606       if (UseCompressedClassPointers)          __ cmpl(tmp, src_klass_addr);
3607       else                   __ cmpptr(tmp, src_klass_addr);
3608       __ jcc(Assembler::equal, known_ok);
3609     } else {
3610       if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3611       else                   __ cmpptr(tmp, dst_klass_addr);
3612       __ jcc(Assembler::equal, known_ok);
3613       __ cmpptr(src, dst);
3614       __ jcc(Assembler::equal, known_ok);
3615     }
3616     __ bind(halt);
3617     __ stop("incorrect type information in arraycopy");
3618     __ bind(known_ok);
3619   }
3620 #endif
3621 
3622 #ifndef PRODUCT
3623   if (PrintC1Statistics) {
3624     __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
3625   }
3626 #endif
3627 
3628 #ifdef _LP64
3629   assert_different_registers(c_rarg0, dst, dst_pos, length);
3630   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3631   assert_different_registers(c_rarg1, length);
3632   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3633   __ mov(c_rarg2, length);
3634 
3635 #else
3636   __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3637   store_parameter(tmp, 0);
3638   __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3639   store_parameter(tmp, 1);
3640   store_parameter(length, 2);
3641 #endif // _LP64
3642 
3643   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3644   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3645   const char *name;
3646   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3647   __ call_VM_leaf(entry, 0);
3648 
3649   __ bind(*stub->continuation());
3650 }
3651 
3652 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3653   assert(op->crc()->is_single_cpu(),  "crc must be register");
3654   assert(op->val()->is_single_cpu(),  "byte value must be register");
3655   assert(op->result_opr()->is_single_cpu(), "result must be register");
3656   Register crc = op->crc()->as_register();
3657   Register val = op->val()->as_register();
3658   Register res = op->result_opr()->as_register();
3659 
3660   assert_different_registers(val, crc, res);
3661 
3662   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
3663   __ notl(crc); // ~crc
3664   __ update_byte_crc32(crc, val, res);
3665   __ notl(crc); // ~crc
3666   __ mov(res, crc);
3667 }
3668 
3669 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3670   Register obj = op->obj_opr()->as_register();  // may not be an oop
3671   Register hdr = op->hdr_opr()->as_register();
3672   Register lock = op->lock_opr()->as_register();
3673   if (!UseFastLocking) {
3674     __ jmp(*op->stub()->entry());
3675   } else if (op->code() == lir_lock) {
3676     Register scratch = noreg;
3677     if (UseBiasedLocking) {
3678       scratch = op->scratch_opr()->as_register();
3679     }
3680     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3681     __ resolve(ACCESS_READ | ACCESS_WRITE, obj);
3682     // add debug info for NullPointerException only if one is possible
3683     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
3684     if (op->info() != NULL) {
3685       add_debug_info_for_null_check(null_check_offset, op->info());
3686     }
3687     // done
3688   } else if (op->code() == lir_unlock) {
3689     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3690     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3691   } else {
3692     Unimplemented();
3693   }
3694   __ bind(*op->stub()->continuation());
3695 }
3696 
3697 
3698 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3699   ciMethod* method = op->profiled_method();
3700   int bci          = op->profiled_bci();
3701   ciMethod* callee = op->profiled_callee();
3702 
3703   // Update counter for all call types
3704   ciMethodData* md = method->method_data_or_null();
3705   assert(md != NULL, "Sanity");
3706   ciProfileData* data = md->bci_to_data(bci);
3707   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
3708   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3709   Register mdo  = op->mdo()->as_register();
3710   __ mov_metadata(mdo, md->constant_encoding());
3711   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3712   // Perform additional virtual call profiling for invokevirtual and
3713   // invokeinterface bytecodes
3714   if (op->should_profile_receiver_type()) {
3715     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3716     Register recv = op->recv()->as_register();
3717     assert_different_registers(mdo, recv);
3718     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3719     ciKlass* known_klass = op->known_holder();
3720     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3721       // We know the type that will be seen at this call site; we can
3722       // statically update the MethodData* rather than needing to do
3723       // dynamic tests on the receiver type
3724 
3725       // NOTE: we should probably put a lock around this search to
3726       // avoid collisions by concurrent compilations
3727       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3728       uint i;
3729       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3730         ciKlass* receiver = vc_data->receiver(i);
3731         if (known_klass->equals(receiver)) {
3732           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3733           __ addptr(data_addr, DataLayout::counter_increment);
3734           return;
3735         }
3736       }
3737 
3738       // Receiver type not found in profile data; select an empty slot
3739 
3740       // Note that this is less efficient than it should be because it
3741       // always does a write to the receiver part of the
3742       // VirtualCallData rather than just the first time
3743       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3744         ciKlass* receiver = vc_data->receiver(i);
3745         if (receiver == NULL) {
3746           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3747           __ mov_metadata(recv_addr, known_klass->constant_encoding());
3748           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3749           __ addptr(data_addr, DataLayout::counter_increment);
3750           return;
3751         }
3752       }
3753     } else {
3754       __ load_klass(recv, recv);
3755       Label update_done;
3756       type_profile_helper(mdo, md, data, recv, &update_done);
3757       // Receiver did not match any saved receiver and there is no empty row for it.
3758       // Increment total counter to indicate polymorphic case.
3759       __ addptr(counter_addr, DataLayout::counter_increment);
3760 
3761       __ bind(update_done);
3762     }
3763   } else {
3764     // Static call
3765     __ addptr(counter_addr, DataLayout::counter_increment);
3766   }
3767 }
3768 
3769 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3770   Register obj = op->obj()->as_register();
3771   Register tmp = op->tmp()->as_pointer_register();
3772   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3773   ciKlass* exact_klass = op->exact_klass();
3774   intptr_t current_klass = op->current_klass();
3775   bool not_null = op->not_null();
3776   bool no_conflict = op->no_conflict();
3777 
3778   Label update, next, none;
3779 
3780   bool do_null = !not_null;
3781   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3782   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3783 
3784   assert(do_null || do_update, "why are we here?");
3785   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3786 
3787   __ verify_oop(obj);
3788 
3789   if (tmp != obj) {
3790     __ mov(tmp, obj);
3791   }
3792   if (do_null) {
3793     __ testptr(tmp, tmp);
3794     __ jccb(Assembler::notZero, update);
3795     if (!TypeEntries::was_null_seen(current_klass)) {
3796       __ orptr(mdo_addr, TypeEntries::null_seen);
3797     }
3798     if (do_update) {
3799 #ifndef ASSERT
3800       __ jmpb(next);
3801     }
3802 #else
3803       __ jmp(next);
3804     }
3805   } else {
3806     __ testptr(tmp, tmp);
3807     __ jcc(Assembler::notZero, update);
3808     __ stop("unexpect null obj");
3809 #endif
3810   }
3811 
3812   __ bind(update);
3813 
3814   if (do_update) {
3815 #ifdef ASSERT
3816     if (exact_klass != NULL) {
3817       Label ok;
3818       __ load_klass(tmp, tmp);
3819       __ push(tmp);
3820       __ mov_metadata(tmp, exact_klass->constant_encoding());
3821       __ cmpptr(tmp, Address(rsp, 0));
3822       __ jcc(Assembler::equal, ok);
3823       __ stop("exact klass and actual klass differ");
3824       __ bind(ok);
3825       __ pop(tmp);
3826     }
3827 #endif
3828     if (!no_conflict) {
3829       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
3830         if (exact_klass != NULL) {
3831           __ mov_metadata(tmp, exact_klass->constant_encoding());
3832         } else {
3833           __ load_klass(tmp, tmp);
3834         }
3835 
3836         __ xorptr(tmp, mdo_addr);
3837         __ testptr(tmp, TypeEntries::type_klass_mask);
3838         // klass seen before, nothing to do. The unknown bit may have been
3839         // set already but no need to check.
3840         __ jccb(Assembler::zero, next);
3841 
3842         __ testptr(tmp, TypeEntries::type_unknown);
3843         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3844 
3845         if (TypeEntries::is_type_none(current_klass)) {
3846           __ cmpptr(mdo_addr, 0);
3847           __ jccb(Assembler::equal, none);
3848           __ cmpptr(mdo_addr, TypeEntries::null_seen);
3849           __ jccb(Assembler::equal, none);
3850           // There is a chance that the checks above (re-reading profiling
3851           // data from memory) fail if another thread has just set the
3852           // profiling to this obj's klass
3853           __ xorptr(tmp, mdo_addr);
3854           __ testptr(tmp, TypeEntries::type_klass_mask);
3855           __ jccb(Assembler::zero, next);
3856         }
3857       } else {
3858         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3859                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3860 
3861         __ movptr(tmp, mdo_addr);
3862         __ testptr(tmp, TypeEntries::type_unknown);
3863         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3864       }
3865 
3866       // different than before. Cannot keep accurate profile.
3867       __ orptr(mdo_addr, TypeEntries::type_unknown);
3868 
3869       if (TypeEntries::is_type_none(current_klass)) {
3870         __ jmpb(next);
3871 
3872         __ bind(none);
3873         // first time here. Set profile type.
3874         __ movptr(mdo_addr, tmp);
3875       }
3876     } else {
3877       // There's a single possible klass at this profile point
3878       assert(exact_klass != NULL, "should be");
3879       if (TypeEntries::is_type_none(current_klass)) {
3880         __ mov_metadata(tmp, exact_klass->constant_encoding());
3881         __ xorptr(tmp, mdo_addr);
3882         __ testptr(tmp, TypeEntries::type_klass_mask);
3883 #ifdef ASSERT
3884         __ jcc(Assembler::zero, next);
3885 
3886         {
3887           Label ok;
3888           __ push(tmp);
3889           __ cmpptr(mdo_addr, 0);
3890           __ jcc(Assembler::equal, ok);
3891           __ cmpptr(mdo_addr, TypeEntries::null_seen);
3892           __ jcc(Assembler::equal, ok);
3893           // may have been set by another thread
3894           __ mov_metadata(tmp, exact_klass->constant_encoding());
3895           __ xorptr(tmp, mdo_addr);
3896           __ testptr(tmp, TypeEntries::type_mask);
3897           __ jcc(Assembler::zero, ok);
3898 
3899           __ stop("unexpected profiling mismatch");
3900           __ bind(ok);
3901           __ pop(tmp);
3902         }
3903 #else
3904         __ jccb(Assembler::zero, next);
3905 #endif
3906         // first time here. Set profile type.
3907         __ movptr(mdo_addr, tmp);
3908       } else {
3909         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3910                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3911 
3912         __ movptr(tmp, mdo_addr);
3913         __ testptr(tmp, TypeEntries::type_unknown);
3914         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3915 
3916         __ orptr(mdo_addr, TypeEntries::type_unknown);
3917       }
3918     }
3919 
3920     __ bind(next);
3921   }
3922 }
3923 
3924 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3925   Unimplemented();
3926 }
3927 
3928 
3929 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3930   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3931 }
3932 
3933 
3934 void LIR_Assembler::align_backward_branch_target() {
3935   __ align(BytesPerWord);
3936 }
3937 
3938 
3939 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3940   if (left->is_single_cpu()) {
3941     __ negl(left->as_register());
3942     move_regs(left->as_register(), dest->as_register());
3943 
3944   } else if (left->is_double_cpu()) {
3945     Register lo = left->as_register_lo();
3946 #ifdef _LP64
3947     Register dst = dest->as_register_lo();
3948     __ movptr(dst, lo);
3949     __ negptr(dst);
3950 #else
3951     Register hi = left->as_register_hi();
3952     __ lneg(hi, lo);
3953     if (dest->as_register_lo() == hi) {
3954       assert(dest->as_register_hi() != lo, "destroying register");
3955       move_regs(hi, dest->as_register_hi());
3956       move_regs(lo, dest->as_register_lo());
3957     } else {
3958       move_regs(lo, dest->as_register_lo());
3959       move_regs(hi, dest->as_register_hi());
3960     }
3961 #endif // _LP64
3962 
3963   } else if (dest->is_single_xmm()) {
3964 #ifdef _LP64
3965     if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3966       assert(tmp->is_valid(), "need temporary");
3967       assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3968       __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3969     }
3970     else
3971 #endif
3972     {
3973       assert(!tmp->is_valid(), "do not need temporary");
3974       if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3975         __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3976       }
3977       __ xorps(dest->as_xmm_float_reg(),
3978                ExternalAddress((address)float_signflip_pool));
3979     }
3980   } else if (dest->is_double_xmm()) {
3981 #ifdef _LP64
3982     if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3983       assert(tmp->is_valid(), "need temporary");
3984       assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3985       __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3986     }
3987     else
3988 #endif
3989     {
3990       assert(!tmp->is_valid(), "do not need temporary");
3991       if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3992         __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3993       }
3994       __ xorpd(dest->as_xmm_double_reg(),
3995                ExternalAddress((address)double_signflip_pool));
3996     }
3997   } else if (left->is_single_fpu() || left->is_double_fpu()) {
3998     assert(left->fpu() == 0, "arg must be on TOS");
3999     assert(dest->fpu() == 0, "dest must be TOS");
4000     __ fchs();
4001 
4002   } else {
4003     ShouldNotReachHere();
4004   }
4005 }
4006 
4007 
4008 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
4009   assert(src->is_address(), "must be an address");
4010   assert(dest->is_register(), "must be a register");
4011 
4012   PatchingStub* patch = NULL;
4013   if (patch_code != lir_patch_none) {
4014     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
4015   }
4016 
4017   Register reg = dest->as_pointer_register();
4018   LIR_Address* addr = src->as_address_ptr();
4019   __ lea(reg, as_Address(addr));
4020 
4021   if (patch != NULL) {
4022     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
4023   }
4024 }
4025 
4026 
4027 
4028 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
4029   assert(!tmp->is_valid(), "don't need temporary");
4030   __ call(RuntimeAddress(dest));
4031   if (info != NULL) {
4032     add_call_info_here(info);
4033   }
4034 }
4035 
4036 
4037 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
4038   assert(type == T_LONG, "only for volatile long fields");
4039 
4040   if (info != NULL) {
4041     add_debug_info_for_null_check_here(info);
4042   }
4043 
4044   if (src->is_double_xmm()) {
4045     if (dest->is_double_cpu()) {
4046 #ifdef _LP64
4047       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
4048 #else
4049       __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
4050       __ psrlq(src->as_xmm_double_reg(), 32);
4051       __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
4052 #endif // _LP64
4053     } else if (dest->is_double_stack()) {
4054       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
4055     } else if (dest->is_address()) {
4056       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
4057     } else {
4058       ShouldNotReachHere();
4059     }
4060 
4061   } else if (dest->is_double_xmm()) {
4062     if (src->is_double_stack()) {
4063       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
4064     } else if (src->is_address()) {
4065       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
4066     } else {
4067       ShouldNotReachHere();
4068     }
4069 
4070   } else if (src->is_double_fpu()) {
4071     assert(src->fpu_regnrLo() == 0, "must be TOS");
4072     if (dest->is_double_stack()) {
4073       __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
4074     } else if (dest->is_address()) {
4075       __ fistp_d(as_Address(dest->as_address_ptr()));
4076     } else {
4077       ShouldNotReachHere();
4078     }
4079 
4080   } else if (dest->is_double_fpu()) {
4081     assert(dest->fpu_regnrLo() == 0, "must be TOS");
4082     if (src->is_double_stack()) {
4083       __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
4084     } else if (src->is_address()) {
4085       __ fild_d(as_Address(src->as_address_ptr()));
4086     } else {
4087       ShouldNotReachHere();
4088     }
4089   } else {
4090     ShouldNotReachHere();
4091   }
4092 }
4093 
4094 #ifdef ASSERT
4095 // emit run-time assertion
4096 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
4097   assert(op->code() == lir_assert, "must be");
4098 
4099   if (op->in_opr1()->is_valid()) {
4100     assert(op->in_opr2()->is_valid(), "both operands must be valid");
4101     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
4102   } else {
4103     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
4104     assert(op->condition() == lir_cond_always, "no other conditions allowed");
4105   }
4106 
4107   Label ok;
4108   if (op->condition() != lir_cond_always) {
4109     Assembler::Condition acond = Assembler::zero;
4110     switch (op->condition()) {
4111       case lir_cond_equal:        acond = Assembler::equal;       break;
4112       case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
4113       case lir_cond_less:         acond = Assembler::less;        break;
4114       case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
4115       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
4116       case lir_cond_greater:      acond = Assembler::greater;     break;
4117       case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
4118       case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
4119       default:                    ShouldNotReachHere();
4120     }
4121     __ jcc(acond, ok);
4122   }
4123   if (op->halt()) {
4124     const char* str = __ code_string(op->msg());
4125     __ stop(str);
4126   } else {
4127     breakpoint();
4128   }
4129   __ bind(ok);
4130 }
4131 #endif
4132 
4133 void LIR_Assembler::membar() {
4134   // QQQ sparc TSO uses this,
4135   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
4136 }
4137 
4138 void LIR_Assembler::membar_acquire() {
4139   // No x86 machines currently require load fences
4140 }
4141 
4142 void LIR_Assembler::membar_release() {
4143   // No x86 machines currently require store fences
4144 }
4145 
4146 void LIR_Assembler::membar_loadload() {
4147   // no-op
4148   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
4149 }
4150 
4151 void LIR_Assembler::membar_storestore() {
4152   // no-op
4153   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
4154 }
4155 
4156 void LIR_Assembler::membar_loadstore() {
4157   // no-op
4158   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
4159 }
4160 
4161 void LIR_Assembler::membar_storeload() {
4162   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4163 }
4164 
4165 void LIR_Assembler::on_spin_wait() {
4166   __ pause ();
4167 }
4168 
4169 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4170   assert(result_reg->is_register(), "check");
4171 #ifdef _LP64
4172   // __ get_thread(result_reg->as_register_lo());
4173   __ mov(result_reg->as_register(), r15_thread);
4174 #else
4175   __ get_thread(result_reg->as_register());
4176 #endif // _LP64
4177 }
4178 
4179 
4180 void LIR_Assembler::peephole(LIR_List*) {
4181   // do nothing for now
4182 }
4183 
4184 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4185   assert(data == dest, "xchg/xadd uses only 2 operands");
4186 
4187   if (data->type() == T_INT) {
4188     if (code == lir_xadd) {
4189       __ lock();
4190       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4191     } else {
4192       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4193     }
4194   } else if (data->is_oop()) {
4195     assert (code == lir_xchg, "xadd for oops");
4196     Register obj = data->as_register();
4197 #ifdef _LP64
4198     if (UseCompressedOops) {
4199       __ encode_heap_oop(obj);
4200       __ xchgl(obj, as_Address(src->as_address_ptr()));
4201       __ decode_heap_oop(obj);
4202     } else {
4203       __ xchgptr(obj, as_Address(src->as_address_ptr()));
4204     }
4205 #else
4206     __ xchgl(obj, as_Address(src->as_address_ptr()));
4207 #endif
4208   } else if (data->type() == T_LONG) {
4209 #ifdef _LP64
4210     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
4211     if (code == lir_xadd) {
4212       __ lock();
4213       __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
4214     } else {
4215       __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
4216     }
4217 #else
4218     ShouldNotReachHere();
4219 #endif
4220   } else {
4221     ShouldNotReachHere();
4222   }
4223 }
4224 
4225 #undef __